From 4f3dc7b8a8f47366df4d0ffb877e9bdd85f6aaf0 Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Fri, 28 Feb 2020 15:54:12 -0800 Subject: [PATCH 01/12] Adding OT Collector metrics exporter --- examples/metrics/collector.py | 53 +++++ examples/metrics/docker/collector-config.yaml | 18 ++ examples/metrics/docker/docker-compose.yaml | 19 ++ examples/metrics/docker/prometheus.yaml | 5 + .../otcollector/metrics_exporter/__init__.py | 172 ++++++++++++++++ .../test_otcollector_metrics_exporter.py | 191 ++++++++++++++++++ ....py => test_otcollector_trace_exporter.py} | 0 7 files changed, 458 insertions(+) create mode 100644 examples/metrics/collector.py create mode 100644 examples/metrics/docker/collector-config.yaml create mode 100644 examples/metrics/docker/docker-compose.yaml create mode 100644 examples/metrics/docker/prometheus.yaml create mode 100644 ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py create mode 100644 ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py rename ext/opentelemetry-ext-otcollector/tests/{test_otcollector_exporter.py => test_otcollector_trace_exporter.py} (100%) diff --git a/examples/metrics/collector.py b/examples/metrics/collector.py new file mode 100644 index 00000000000..42c88b6d800 --- /dev/null +++ b/examples/metrics/collector.py @@ -0,0 +1,53 @@ +# Copyright 2020, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +This module serves as an example for a simple application using metrics +exporting to Collector +""" + +from opentelemetry import metrics +from opentelemetry.ext.otcollector.metrics_exporter import ( + CollectorMetricsExporter, +) +from opentelemetry.sdk.metrics import Counter, Meter +from opentelemetry.sdk.metrics.export.controller import PushController + +# Meter is responsible for creating and recording metrics +metrics.set_preferred_meter_implementation(lambda _: Meter()) +meter = metrics.meter() +# exporter to export metrics to OT Collector +exporter = CollectorMetricsExporter( + service_name="basic-service", endpoint="localhost:55678" +) +# controller collects metrics created from meter and exports it via the +# exporter every interval +controller = PushController(meter, exporter, 5) + +counter = meter.create_metric( + "requests", + "number of requests", + "requests", + int, + Counter, + ("environment",), +) + +# Labelsets are used to identify key-values that are associated with a specific +# metric that you want to record. These are useful for pre-aggregation and can +# be used to store custom dimensions pertaining to a metric +label_set = meter.get_label_set({"environment": "staging"}) + +counter.add(25, label_set) +input("Press any key to exit...") diff --git a/examples/metrics/docker/collector-config.yaml b/examples/metrics/docker/collector-config.yaml new file mode 100644 index 00000000000..78f685d208e --- /dev/null +++ b/examples/metrics/docker/collector-config.yaml @@ -0,0 +1,18 @@ +receivers: + opencensus: + endpoint: "0.0.0.0:55678" + +exporters: + prometheus: + endpoint: "0.0.0.0:8889" + logging: {} + +processors: + batch: + queued_retry: + +service: + pipelines: + metrics: + receivers: [opencensus] + exporters: [logging, prometheus] diff --git a/examples/metrics/docker/docker-compose.yaml b/examples/metrics/docker/docker-compose.yaml new file mode 100644 index 00000000000..d12a0a4f18c --- /dev/null +++ b/examples/metrics/docker/docker-compose.yaml @@ -0,0 +1,19 @@ +version: "2" +services: + + otel-collector: + image: omnition/opentelemetry-collector-contrib:latest + command: ["--config=/conf/collector-config.yaml", "--log-level=DEBUG"] + volumes: + - ./collector-config.yaml:/conf/collector-config.yaml + ports: + - "8889:8889" # Prometheus exporter metrics + - "55678:55678" # OpenCensus receiver + + prometheus: + container_name: prometheus + image: prom/prometheus:latest + volumes: + - ./prometheus.yaml:/etc/prometheus/prometheus.yml + ports: + - "9090:9090" diff --git a/examples/metrics/docker/prometheus.yaml b/examples/metrics/docker/prometheus.yaml new file mode 100644 index 00000000000..4eb23572314 --- /dev/null +++ b/examples/metrics/docker/prometheus.yaml @@ -0,0 +1,5 @@ +scrape_configs: + - job_name: 'otel-collector' + scrape_interval: 5s + static_configs: + - targets: ['otel-collector:8889'] diff --git a/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py b/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py new file mode 100644 index 00000000000..91f38631fc0 --- /dev/null +++ b/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py @@ -0,0 +1,172 @@ +# Copyright 2020, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""OpenTelemetry Collector Metrics Exporter.""" + +import logging +from typing import Optional, Sequence + +import grpc +from opencensus.proto.agent.metrics.v1 import ( + metrics_service_pb2, + metrics_service_pb2_grpc, +) +from opencensus.proto.metrics.v1 import metrics_pb2 + +import opentelemetry.ext.otcollector.util as utils +from opentelemetry.sdk.metrics import Counter, Gauge, Measure, Meter, Metric +from opentelemetry.sdk.metrics.export import ( + MetricRecord, + MetricsExporter, + MetricsExportResult, + aggregate, +) + +DEFAULT_ENDPOINT = "localhost:55678" + +logger = logging.getLogger(__name__) + + +# pylint: disable=no-member +class CollectorMetricsExporter(MetricsExporter): + """OpenTelemetry Collector metrics exporter. + + Args: + endpoint: OpenTelemetry Collector OpenCensus receiver endpoint. + service_name: Name of Collector service. + host_name: Host name. + client: MetricsService client stub. + """ + + def __init__( + self, + endpoint=DEFAULT_ENDPOINT, + service_name=None, + host_name=None, + client=None, + ): + self.endpoint = endpoint + if client is None: + self.channel = grpc.insecure_channel(self.endpoint) + self.client = metrics_service_pb2_grpc.MetricsServiceStub( + channel=self.channel + ) + else: + self.client = client + + self.node = utils.get_node(service_name, host_name) + + def export( + self, metric_records: Sequence[MetricRecord] + ) -> MetricsExportResult: + try: + responses = self.client.Export( + self.generate_metrics_requests(metric_records) + ) + + # Read response + for _ in responses: + pass + + except grpc.RpcError: + return MetricsExportResult.FAILED_RETRYABLE + + return MetricsExportResult.SUCCESS + + def shutdown(self) -> None: + pass + + def generate_metrics_requests(self, metrics): + collector_metrics = translate_to_collector(metrics) + service_request = metrics_service_pb2.ExportMetricsServiceRequest( + node=self.node, metrics=collector_metrics + ) + yield service_request + + +# pylint: disable=too-many-branches +def translate_to_collector(metric_records: Sequence[MetricRecord]): + collector_metrics = [] + for metric_record in metric_records: + + label_values = [] + label_keys = [] + for label_tuple in metric_record.label_set.labels: + label_keys.append(metrics_pb2.LabelKey(key=label_tuple[0])) + label_values.append( + metrics_pb2.LabelValue( + has_value=label_tuple[1] is not None, value=label_tuple[1] + ) + ) + + metric_descriptor = metrics_pb2.MetricDescriptor( + name=metric_record.metric.name, + description=metric_record.metric.description, + unit=metric_record.metric.unit, + type=get_collector_metric_type(metric_record.metric), + label_keys=label_keys, + ) + + timeseries = metrics_pb2.TimeSeries( + start_timestamp=utils.proto_timestamp_from_time_ns( + metric_record.metric.get_handle( + metric_record.label_set + ).last_update_timestamp + ), + label_values=label_values, + points=[get_collector_point(metric_record)], + ) + collector_metrics.append( + metrics_pb2.Metric( + metric_descriptor=metric_descriptor, timeseries=[timeseries] + ) + ) + return collector_metrics + + +# pylint: disable=no-else-return +def get_collector_metric_type(metric: Metric): + if isinstance(metric, Counter): + if metric.value_type == int: + return metrics_pb2.MetricDescriptor.CUMULATIVE_INT64 + elif metric.value_type == float: + return metrics_pb2.MetricDescriptor.CUMULATIVE_DOUBLE + if isinstance(metric, Gauge): + if metric.value_type == int: + return metrics_pb2.MetricDescriptor.GAUGE_INT64 + elif metric.value_type == float: + return metrics_pb2.MetricDescriptor.GAUGE_DOUBLE + + return metrics_pb2.MetricDescriptor.UNSPECIFIED + + +def get_collector_point(metric_record: MetricRecord): + point = metrics_pb2.Point( + timestamp=utils.proto_timestamp_from_time_ns( + metric_record.metric.get_handle( + metric_record.label_set + ).last_update_timestamp + ) + ) + if metric_record.metric.value_type == int: + point.int64_value = metric_record.aggregator.checkpoint + elif metric_record.metric.value_type == float: + point.double_value = metric_record.aggregator.checkpoint + else: + raise TypeError( + "Unsupported metric type: {}".format( + metric_record.metric.value_type + ) + ) + return point diff --git a/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py new file mode 100644 index 00000000000..38e61417e13 --- /dev/null +++ b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py @@ -0,0 +1,191 @@ +# Copyright 2020, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from unittest import mock + +import grpc +from google.protobuf.timestamp_pb2 import Timestamp +from opencensus.proto.metrics.v1 import metrics_pb2 + +from opentelemetry import metrics +from opentelemetry.ext.otcollector import metrics_exporter +from opentelemetry.sdk.metrics import Counter, Gauge, Measure, Meter +from opentelemetry.sdk.metrics.export import ( + MetricRecord, + MetricsExportResult, + aggregate, +) + + +# pylint: disable=no-member +class TestCollectorMetricsExporter(unittest.TestCase): + @classmethod + def setUpClass(cls): + metrics.set_preferred_meter_implementation(lambda _: Meter()) + cls._meter = metrics.meter() + kvp = {"environment": "staging"} + cls._test_label_set = cls._meter.get_label_set(kvp) + + def test_constructor(self): + mock_get_node = mock.Mock() + patch = mock.patch( + "opentelemetry.ext.otcollector.util.get_node", + side_effect=mock_get_node, + ) + service_name = "testServiceName" + host_name = "testHostName" + client = grpc.insecure_channel("") + endpoint = "testEndpoint" + with patch: + exporter = metrics_exporter.CollectorMetricsExporter( + service_name=service_name, + host_name=host_name, + endpoint=endpoint, + client=client, + ) + + self.assertIs(exporter.client, client) + self.assertEqual(exporter.endpoint, endpoint) + mock_get_node.assert_called_with(service_name, host_name) + + def test_get_collector_metric_type(self): + result = metrics_exporter.get_collector_metric_type( + Counter("testName", "testDescription", "unit", int, None) + ) + self.assertIs(result, metrics_pb2.MetricDescriptor.CUMULATIVE_INT64) + result = metrics_exporter.get_collector_metric_type( + Counter("testName", "testDescription", "unit", float, None) + ) + self.assertIs(result, metrics_pb2.MetricDescriptor.CUMULATIVE_DOUBLE) + result = metrics_exporter.get_collector_metric_type( + Gauge("testName", "testDescription", "unit", int, None) + ) + self.assertIs(result, metrics_pb2.MetricDescriptor.GAUGE_INT64) + result = metrics_exporter.get_collector_metric_type( + Gauge("testName", "testDescription", "unit", float, None) + ) + self.assertIs(result, metrics_pb2.MetricDescriptor.GAUGE_DOUBLE) + result = metrics_exporter.get_collector_metric_type( + Measure("testName", "testDescription", "unit", None, None) + ) + self.assertIs(result, metrics_pb2.MetricDescriptor.UNSPECIFIED) + + def test_get_collector_point(self): + aggregator = aggregate.CounterAggregator() + label_set = self._meter.get_label_set({"environment": "staging"}) + int_counter = self._meter.create_metric( + "testName", "testDescription", "unit", int, Counter + ) + float_counter = self._meter.create_metric( + "testName", "testDescription", "unit", float, Counter + ) + measure = self._meter.create_metric( + "testName", "testDescription", "unit", float, Measure + ) + result = metrics_exporter.get_collector_point( + MetricRecord(aggregator, label_set, int_counter) + ) + self.assertIsInstance(result, metrics_pb2.Point) + self.assertIsInstance(result.timestamp, Timestamp) + self.assertEqual(result.int64_value, 0) + aggregator.update(123.5) + aggregator.take_checkpoint() + result = metrics_exporter.get_collector_point( + MetricRecord(aggregator, label_set, float_counter) + ) + self.assertEqual(result.double_value, 123.5) + self.assertRaises( + TypeError, + metrics_exporter.get_collector_point( + MetricRecord(aggregator, label_set, measure) + ), + ) + + def test_export(self): + mock_client = mock.MagicMock() + mock_export = mock.MagicMock() + mock_client.Export = mock_export + host_name = "testHostName" + collector_exporter = metrics_exporter.CollectorMetricsExporter( + client=mock_client, host_name=host_name + ) + test_metric = self._meter.create_metric( + "testname", "testdesc", "unit", int, Counter, ["environment"] + ) + record = MetricRecord( + aggregate.CounterAggregator(), self._test_label_set, test_metric + ) + + result = collector_exporter.export([record]) + self.assertIs(result, MetricsExportResult.SUCCESS) + # pylint: disable=unsubscriptable-object + export_arg = mock_export.call_args[0] + service_request = next(export_arg[0]) + output_metrics = getattr(service_request, "metrics") + output_node = getattr(service_request, "node") + self.assertEqual(len(output_metrics), 1) + self.assertIsNotNone(getattr(output_node, "library_info")) + self.assertIsNotNone(getattr(output_node, "service_info")) + output_identifier = getattr(output_node, "identifier") + self.assertEqual( + getattr(output_identifier, "host_name"), "testHostName" + ) + + def test_translate_to_collector(self): + + test_metric = self._meter.create_metric( + "testname", "testdesc", "unit", int, Counter, ["environment"] + ) + aggregator = aggregate.CounterAggregator() + aggregator.update(123) + aggregator.take_checkpoint() + record = MetricRecord(aggregator, self._test_label_set, test_metric) + output_metrics = metrics_exporter.translate_to_collector([record]) + self.assertEqual(len(output_metrics), 1) + self.assertIsInstance(output_metrics[0], metrics_pb2.Metric) + self.assertEqual(output_metrics[0].metric_descriptor.name, "testname") + self.assertEqual( + output_metrics[0].metric_descriptor.description, "testdesc" + ) + self.assertEqual(output_metrics[0].metric_descriptor.unit, "unit") + self.assertEqual( + output_metrics[0].metric_descriptor.type, + metrics_pb2.MetricDescriptor.CUMULATIVE_INT64, + ) + self.assertEqual( + len(output_metrics[0].metric_descriptor.label_keys), 1 + ) + self.assertEqual( + output_metrics[0].metric_descriptor.label_keys[0].key, + "environment", + ) + self.assertEqual(len(output_metrics[0].timeseries), 1) + self.assertGreater( + output_metrics[0].timeseries[0].start_timestamp.nanos, 0 + ) + self.assertEqual(len(output_metrics[0].timeseries[0].label_values), 1) + self.assertEqual( + output_metrics[0].timeseries[0].label_values[0].has_value, True + ) + self.assertEqual( + output_metrics[0].timeseries[0].label_values[0].value, "staging" + ) + self.assertEqual(len(output_metrics[0].timeseries[0].points), 1) + self.assertGreater( + output_metrics[0].timeseries[0].points[0].timestamp.nanos, 0 + ) + self.assertEqual( + output_metrics[0].timeseries[0].points[0].int64_value, 123 + ) diff --git a/ext/opentelemetry-ext-otcollector/tests/test_otcollector_exporter.py b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_trace_exporter.py similarity index 100% rename from ext/opentelemetry-ext-otcollector/tests/test_otcollector_exporter.py rename to ext/opentelemetry-ext-otcollector/tests/test_otcollector_trace_exporter.py From d662d89778d4c6db078849503c4628a211f56420 Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Fri, 28 Feb 2020 16:19:54 -0800 Subject: [PATCH 02/12] Fix format --- .../opentelemetry/ext/otcollector/metrics_exporter/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py b/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py index 91f38631fc0..0174297e147 100644 --- a/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py +++ b/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py @@ -140,7 +140,7 @@ def get_collector_metric_type(metric: Metric): if isinstance(metric, Counter): if metric.value_type == int: return metrics_pb2.MetricDescriptor.CUMULATIVE_INT64 - elif metric.value_type == float: + elif metric.value_type == float: return metrics_pb2.MetricDescriptor.CUMULATIVE_DOUBLE if isinstance(metric, Gauge): if metric.value_type == int: From ceafe10a04c53716786202721c41aa05183689cc Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Mon, 2 Mar 2020 12:15:32 -0800 Subject: [PATCH 03/12] Addressing comments --- examples/metrics/collector.py | 6 +++--- .../ext/otcollector/metrics_exporter/__init__.py | 13 +++++-------- .../tests/test_otcollector_metrics_exporter.py | 3 --- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/examples/metrics/collector.py b/examples/metrics/collector.py index 42c88b6d800..230b2963934 100644 --- a/examples/metrics/collector.py +++ b/examples/metrics/collector.py @@ -21,12 +21,12 @@ from opentelemetry.ext.otcollector.metrics_exporter import ( CollectorMetricsExporter, ) -from opentelemetry.sdk.metrics import Counter, Meter +from opentelemetry.sdk.metrics import Counter, MeterProvider from opentelemetry.sdk.metrics.export.controller import PushController # Meter is responsible for creating and recording metrics -metrics.set_preferred_meter_implementation(lambda _: Meter()) -meter = metrics.meter() +metrics.set_preferred_meter_provider_implementation(lambda _: MeterProvider()) +meter = metrics.get_meter(__name__) # exporter to export metrics to OT Collector exporter = CollectorMetricsExporter( service_name="basic-service", endpoint="localhost:55678" diff --git a/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py b/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py index 0174297e147..1c5322c52a2 100644 --- a/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py +++ b/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py @@ -96,7 +96,9 @@ def generate_metrics_requests(self, metrics): # pylint: disable=too-many-branches -def translate_to_collector(metric_records: Sequence[MetricRecord]): +def translate_to_collector( + metric_records: Sequence[MetricRecord] +) -> Sequence[metrics_pb2.Metric]: collector_metrics = [] for metric_record in metric_records: @@ -119,11 +121,6 @@ def translate_to_collector(metric_records: Sequence[MetricRecord]): ) timeseries = metrics_pb2.TimeSeries( - start_timestamp=utils.proto_timestamp_from_time_ns( - metric_record.metric.get_handle( - metric_record.label_set - ).last_update_timestamp - ), label_values=label_values, points=[get_collector_point(metric_record)], ) @@ -136,7 +133,7 @@ def translate_to_collector(metric_records: Sequence[MetricRecord]): # pylint: disable=no-else-return -def get_collector_metric_type(metric: Metric): +def get_collector_metric_type(metric: Metric) -> metrics_pb2.MetricDescriptor: if isinstance(metric, Counter): if metric.value_type == int: return metrics_pb2.MetricDescriptor.CUMULATIVE_INT64 @@ -151,7 +148,7 @@ def get_collector_metric_type(metric: Metric): return metrics_pb2.MetricDescriptor.UNSPECIFIED -def get_collector_point(metric_record: MetricRecord): +def get_collector_point(metric_record: MetricRecord) -> metrics_pb2.Point: point = metrics_pb2.Point( timestamp=utils.proto_timestamp_from_time_ns( metric_record.metric.get_handle( diff --git a/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py index 38e61417e13..c864916596a 100644 --- a/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py +++ b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py @@ -172,9 +172,6 @@ def test_translate_to_collector(self): "environment", ) self.assertEqual(len(output_metrics[0].timeseries), 1) - self.assertGreater( - output_metrics[0].timeseries[0].start_timestamp.nanos, 0 - ) self.assertEqual(len(output_metrics[0].timeseries[0].label_values), 1) self.assertEqual( output_metrics[0].timeseries[0].label_values[0].has_value, True From dd03d6e88baee29bca8587dd7fa976ebb892845f Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Mon, 2 Mar 2020 14:11:11 -0800 Subject: [PATCH 04/12] Fixing tests --- .../tests/test_otcollector_metrics_exporter.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py index c864916596a..b0264cf12a9 100644 --- a/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py +++ b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py @@ -21,7 +21,7 @@ from opentelemetry import metrics from opentelemetry.ext.otcollector import metrics_exporter -from opentelemetry.sdk.metrics import Counter, Gauge, Measure, Meter +from opentelemetry.sdk.metrics import Counter, Gauge, Measure, MeterProvider from opentelemetry.sdk.metrics.export import ( MetricRecord, MetricsExportResult, @@ -33,8 +33,10 @@ class TestCollectorMetricsExporter(unittest.TestCase): @classmethod def setUpClass(cls): - metrics.set_preferred_meter_implementation(lambda _: Meter()) - cls._meter = metrics.meter() + metrics.set_preferred_meter_provider_implementation( + lambda _: MeterProvider() + ) + cls._meter = metrics.get_meter(__name__) kvp = {"environment": "staging"} cls._test_label_set = cls._meter.get_label_set(kvp) From 9d8ffa82ecb5eff42d6c991aa74660b897cf5301 Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Mon, 2 Mar 2020 14:34:16 -0800 Subject: [PATCH 05/12] Removing Gauge --- .../ext/otcollector/metrics_exporter/__init__.py | 8 +------- .../tests/test_otcollector_metrics_exporter.py | 10 +--------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py b/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py index 1c5322c52a2..31bda434f13 100644 --- a/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py +++ b/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py @@ -25,7 +25,7 @@ from opencensus.proto.metrics.v1 import metrics_pb2 import opentelemetry.ext.otcollector.util as utils -from opentelemetry.sdk.metrics import Counter, Gauge, Measure, Meter, Metric +from opentelemetry.sdk.metrics import Counter, Measure, Meter, Metric from opentelemetry.sdk.metrics.export import ( MetricRecord, MetricsExporter, @@ -139,12 +139,6 @@ def get_collector_metric_type(metric: Metric) -> metrics_pb2.MetricDescriptor: return metrics_pb2.MetricDescriptor.CUMULATIVE_INT64 elif metric.value_type == float: return metrics_pb2.MetricDescriptor.CUMULATIVE_DOUBLE - if isinstance(metric, Gauge): - if metric.value_type == int: - return metrics_pb2.MetricDescriptor.GAUGE_INT64 - elif metric.value_type == float: - return metrics_pb2.MetricDescriptor.GAUGE_DOUBLE - return metrics_pb2.MetricDescriptor.UNSPECIFIED diff --git a/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py index b0264cf12a9..9160e31aa61 100644 --- a/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py +++ b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py @@ -21,7 +21,7 @@ from opentelemetry import metrics from opentelemetry.ext.otcollector import metrics_exporter -from opentelemetry.sdk.metrics import Counter, Gauge, Measure, MeterProvider +from opentelemetry.sdk.metrics import Counter, Measure, MeterProvider from opentelemetry.sdk.metrics.export import ( MetricRecord, MetricsExportResult, @@ -71,14 +71,6 @@ def test_get_collector_metric_type(self): Counter("testName", "testDescription", "unit", float, None) ) self.assertIs(result, metrics_pb2.MetricDescriptor.CUMULATIVE_DOUBLE) - result = metrics_exporter.get_collector_metric_type( - Gauge("testName", "testDescription", "unit", int, None) - ) - self.assertIs(result, metrics_pb2.MetricDescriptor.GAUGE_INT64) - result = metrics_exporter.get_collector_metric_type( - Gauge("testName", "testDescription", "unit", float, None) - ) - self.assertIs(result, metrics_pb2.MetricDescriptor.GAUGE_DOUBLE) result = metrics_exporter.get_collector_metric_type( Measure("testName", "testDescription", "unit", None, None) ) From d1b6fe046914aa9722d46e9780a9f25bfbf0c855 Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Mon, 2 Mar 2020 14:59:05 -0800 Subject: [PATCH 06/12] Fixing format issue --- .../opentelemetry/ext/otcollector/metrics_exporter/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py b/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py index 31bda434f13..9125bc70cb7 100644 --- a/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py +++ b/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py @@ -97,7 +97,7 @@ def generate_metrics_requests(self, metrics): # pylint: disable=too-many-branches def translate_to_collector( - metric_records: Sequence[MetricRecord] + metric_records: Sequence[MetricRecord], ) -> Sequence[metrics_pb2.Metric]: collector_metrics = [] for metric_record in metric_records: From 194f04d202a756af5e689b7a7a20220c7fc36640 Mon Sep 17 00:00:00 2001 From: Hector Hernandez Date: Mon, 9 Mar 2020 11:14:53 -0700 Subject: [PATCH 07/12] Updating readme Addressing comments --- ext/opentelemetry-ext-otcollector/README.rst | 47 ++++++++++++++++++- .../test_otcollector_metrics_exporter.py | 11 ++++- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/ext/opentelemetry-ext-otcollector/README.rst b/ext/opentelemetry-ext-otcollector/README.rst index 33d8d587479..a76813989cd 100644 --- a/ext/opentelemetry-ext-otcollector/README.rst +++ b/ext/opentelemetry-ext-otcollector/README.rst @@ -6,7 +6,7 @@ OpenTelemetry Collector Exporter .. |pypi| image:: https://badge.fury.io/py/opentelemetry-ext-otcollector.svg :target: https://pypi.org/project/opentelemetry-ext-otcollector/ -This library allows to export data to `OpenTelemetry Collector `_. +This library allows to export data to `OpenTelemetry Collector `_ , currently using OpenCensus receiver in Collector side. Installation ------------ @@ -16,7 +16,7 @@ Installation pip install opentelemetry-ext-otcollector -Usage +Traces Usage ----- The **OpenTelemetry Collector Exporter** allows to export `OpenTelemetry`_ traces to `OpenTelemetry Collector`_. @@ -48,6 +48,49 @@ The **OpenTelemetry Collector Exporter** allows to export `OpenTelemetry`_ trace with tracer.start_as_current_span("foo"): print("Hello world!") +Metrics Usage +----- + +The **OpenTelemetry Collector Exporter** allows to export `OpenTelemetry`_ metrics to `OpenTelemetry Collector`_. + +.. code:: python + + from opentelemetry import metrics + from opentelemetry.ext.otcollector.metrics_exporter import CollectorMetricsExporter + from opentelemetry.sdk.metrics import Counter, MeterProvider + from opentelemetry.sdk.metrics.export.controller import PushController + + + # create a CollectorMetricsExporter + collector_exporter = CollectorMetricsExporter( + # optional: + # endpoint="myCollectorUrl:55678", + # service_name="test_service", + # host_name="machine/container name", + ) + + # Meter is responsible for creating and recording metrics + metrics.set_preferred_meter_provider_implementation(lambda _: MeterProvider()) + meter = metrics.get_meter(__name__) + # controller collects metrics created from meter and exports it via the + # exporter every interval + controller = PushController(meter, exporter, 5) + counter = meter.create_metric( + "requests", + "number of requests", + "requests", + int, + Counter, + ("environment",), + ) + # Labelsets are used to identify key-values that are associated with a specific + # metric that you want to record. These are useful for pre-aggregation and can + # be used to store custom dimensions pertaining to a metric + label_set = meter.get_label_set({"environment": "staging"}) + + counter.add(25, label_set) + + References ---------- diff --git a/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py index 9160e31aa61..ac11b6cff5b 100644 --- a/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py +++ b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py @@ -174,8 +174,15 @@ def test_translate_to_collector(self): output_metrics[0].timeseries[0].label_values[0].value, "staging" ) self.assertEqual(len(output_metrics[0].timeseries[0].points), 1) - self.assertGreater( - output_metrics[0].timeseries[0].points[0].timestamp.nanos, 0 + self.assertEqual( + output_metrics[0].timeseries[0].points[0].timestamp.seconds, + record.metric.get_handle(record.label_set).last_update_timestamp + // 1000000000, + ) + self.assertEqual( + output_metrics[0].timeseries[0].points[0].timestamp.nanos, + record.metric.get_handle(record.label_set).last_update_timestamp + % 1000000000, ) self.assertEqual( output_metrics[0].timeseries[0].points[0].int64_value, 123 From 839770d44177e403b815d88608956233af4ca3e2 Mon Sep 17 00:00:00 2001 From: Hector Hernandez <39923391+hectorhdzg@users.noreply.github.com> Date: Mon, 9 Mar 2020 15:12:43 -0700 Subject: [PATCH 08/12] Apply suggestions from code review Committing suggestions Co-Authored-By: Chris Kleinknecht --- ext/opentelemetry-ext-otcollector/README.rst | 6 +++--- .../ext/otcollector/metrics_exporter/__init__.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ext/opentelemetry-ext-otcollector/README.rst b/ext/opentelemetry-ext-otcollector/README.rst index a76813989cd..200ec9a91d0 100644 --- a/ext/opentelemetry-ext-otcollector/README.rst +++ b/ext/opentelemetry-ext-otcollector/README.rst @@ -17,7 +17,7 @@ Installation Traces Usage ------ +------------ The **OpenTelemetry Collector Exporter** allows to export `OpenTelemetry`_ traces to `OpenTelemetry Collector`_. @@ -49,7 +49,7 @@ The **OpenTelemetry Collector Exporter** allows to export `OpenTelemetry`_ trace print("Hello world!") Metrics Usage ------ +------------- The **OpenTelemetry Collector Exporter** allows to export `OpenTelemetry`_ metrics to `OpenTelemetry Collector`_. @@ -74,7 +74,7 @@ The **OpenTelemetry Collector Exporter** allows to export `OpenTelemetry`_ metri meter = metrics.get_meter(__name__) # controller collects metrics created from meter and exports it via the # exporter every interval - controller = PushController(meter, exporter, 5) + controller = PushController(meter, collector_exporter, 5) counter = meter.create_metric( "requests", "number of requests", diff --git a/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py b/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py index 9125bc70cb7..565616869f6 100644 --- a/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py +++ b/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py @@ -15,7 +15,7 @@ """OpenTelemetry Collector Metrics Exporter.""" import logging -from typing import Optional, Sequence +from typing import Sequence import grpc from opencensus.proto.agent.metrics.v1 import ( @@ -25,7 +25,7 @@ from opencensus.proto.metrics.v1 import metrics_pb2 import opentelemetry.ext.otcollector.util as utils -from opentelemetry.sdk.metrics import Counter, Measure, Meter, Metric +from opentelemetry.sdk.metrics import Counter, Metric from opentelemetry.sdk.metrics.export import ( MetricRecord, MetricsExporter, From 3993da987adb5a41543da072a1a96c4cf992813d Mon Sep 17 00:00:00 2001 From: Hector Hernandez Date: Mon, 9 Mar 2020 15:41:35 -0700 Subject: [PATCH 09/12] Addressing comments --- .../ext/otcollector/metrics_exporter/__init__.py | 16 +++++++++------- .../tests/test_otcollector_metrics_exporter.py | 10 ++++++++++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py b/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py index 565616869f6..aa9e3239be7 100644 --- a/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py +++ b/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py @@ -51,16 +51,16 @@ class CollectorMetricsExporter(MetricsExporter): def __init__( self, - endpoint=DEFAULT_ENDPOINT, - service_name=None, - host_name=None, - client=None, + endpoint: str = DEFAULT_ENDPOINT, + service_name: str = None, + host_name: str = None, + client: metrics_service_pb2_grpc.MetricsServiceStub = None, ): self.endpoint = endpoint if client is None: - self.channel = grpc.insecure_channel(self.endpoint) + channel = grpc.insecure_channel(self.endpoint) self.client = metrics_service_pb2_grpc.MetricsServiceStub( - channel=self.channel + channel=channel ) else: self.client = client @@ -87,7 +87,9 @@ def export( def shutdown(self) -> None: pass - def generate_metrics_requests(self, metrics): + def generate_metrics_requests( + self, metrics: Sequence[MetricRecord] + ) -> metrics_service_pb2.ExportMetricsServiceRequest: collector_metrics = translate_to_collector(metrics) service_request = metrics_service_pb2.ExportMetricsServiceRequest( node=self.node, metrics=collector_metrics diff --git a/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py index ac11b6cff5b..df85a683950 100644 --- a/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py +++ b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py @@ -33,6 +33,10 @@ class TestCollectorMetricsExporter(unittest.TestCase): @classmethod def setUpClass(cls): + cls._meter_defaults = ( + metrics._METER_PROVIDER, + metrics._METER_PROVIDER_FACTORY, + ) metrics.set_preferred_meter_provider_implementation( lambda _: MeterProvider() ) @@ -40,6 +44,12 @@ def setUpClass(cls): kvp = {"environment": "staging"} cls._test_label_set = cls._meter.get_label_set(kvp) + @classmethod + def tearDownClass(cls): + metrics._METER_PROVIDER, metrics._METER_PROVIDER_FACTORY = ( + cls._meter_defaults + ) + def test_constructor(self): mock_get_node = mock.Mock() patch = mock.patch( From b690dbd58cad10d4b4c941c7f1bbd5990f339fc7 Mon Sep 17 00:00:00 2001 From: Chris Kleinknecht Date: Mon, 9 Mar 2020 23:24:36 -0700 Subject: [PATCH 10/12] Ignore protected-access lint error in shim test --- .../tests/test_otcollector_metrics_exporter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py index df85a683950..0f3d18fdec6 100644 --- a/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py +++ b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py @@ -33,6 +33,7 @@ class TestCollectorMetricsExporter(unittest.TestCase): @classmethod def setUpClass(cls): + # pylint: disable=protected-access cls._meter_defaults = ( metrics._METER_PROVIDER, metrics._METER_PROVIDER_FACTORY, @@ -46,6 +47,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): + # pylint: disable=protected-access metrics._METER_PROVIDER, metrics._METER_PROVIDER_FACTORY = ( cls._meter_defaults ) From b910e1ef10933ad4fe0a24d76baa1c01fe1695a0 Mon Sep 17 00:00:00 2001 From: Chris Kleinknecht Date: Tue, 10 Mar 2020 09:53:41 -0700 Subject: [PATCH 11/12] Blacken --- .../tests/test_otcollector_metrics_exporter.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py index 0f3d18fdec6..fc2799dd809 100644 --- a/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py +++ b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py @@ -48,9 +48,10 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): # pylint: disable=protected-access - metrics._METER_PROVIDER, metrics._METER_PROVIDER_FACTORY = ( - cls._meter_defaults - ) + ( + metrics._METER_PROVIDER, + metrics._METER_PROVIDER_FACTORY, + ) = cls._meter_defaults def test_constructor(self): mock_get_node = mock.Mock() From 22437e10a1cc771a3b74c541f624f8637d588b00 Mon Sep 17 00:00:00 2001 From: Hector Hernandez Date: Wed, 11 Mar 2020 10:36:49 -0700 Subject: [PATCH 12/12] Updating name of handlers --- .../ext/otcollector/metrics_exporter/__init__.py | 2 +- .../tests/test_otcollector_metrics_exporter.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py b/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py index aa9e3239be7..12715035c25 100644 --- a/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py +++ b/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py @@ -147,7 +147,7 @@ def get_collector_metric_type(metric: Metric) -> metrics_pb2.MetricDescriptor: def get_collector_point(metric_record: MetricRecord) -> metrics_pb2.Point: point = metrics_pb2.Point( timestamp=utils.proto_timestamp_from_time_ns( - metric_record.metric.get_handle( + metric_record.metric.bind( metric_record.label_set ).last_update_timestamp ) diff --git a/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py index fc2799dd809..f58f9768d60 100644 --- a/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py +++ b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py @@ -189,12 +189,12 @@ def test_translate_to_collector(self): self.assertEqual(len(output_metrics[0].timeseries[0].points), 1) self.assertEqual( output_metrics[0].timeseries[0].points[0].timestamp.seconds, - record.metric.get_handle(record.label_set).last_update_timestamp + record.metric.bind(record.label_set).last_update_timestamp // 1000000000, ) self.assertEqual( output_metrics[0].timeseries[0].points[0].timestamp.nanos, - record.metric.get_handle(record.label_set).last_update_timestamp + record.metric.bind(record.label_set).last_update_timestamp % 1000000000, ) self.assertEqual(