diff --git a/examples/metrics/collector.py b/examples/metrics/collector.py new file mode 100644 index 00000000000..230b2963934 --- /dev/null +++ b/examples/metrics/collector.py @@ -0,0 +1,53 @@ +# Copyright 2020, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +This module serves as an example for a simple application using metrics +exporting to Collector +""" + +from opentelemetry import metrics +from opentelemetry.ext.otcollector.metrics_exporter import ( + CollectorMetricsExporter, +) +from opentelemetry.sdk.metrics import Counter, MeterProvider +from opentelemetry.sdk.metrics.export.controller import PushController + +# Meter is responsible for creating and recording metrics +metrics.set_preferred_meter_provider_implementation(lambda _: MeterProvider()) +meter = metrics.get_meter(__name__) +# exporter to export metrics to OT Collector +exporter = CollectorMetricsExporter( + service_name="basic-service", endpoint="localhost:55678" +) +# controller collects metrics created from meter and exports it via the +# exporter every interval +controller = PushController(meter, exporter, 5) + +counter = meter.create_metric( + "requests", + "number of requests", + "requests", + int, + Counter, + ("environment",), +) + +# Labelsets are used to identify key-values that are associated with a specific +# metric that you want to record. These are useful for pre-aggregation and can +# be used to store custom dimensions pertaining to a metric +label_set = meter.get_label_set({"environment": "staging"}) + +counter.add(25, label_set) +input("Press any key to exit...") diff --git a/examples/metrics/docker/collector-config.yaml b/examples/metrics/docker/collector-config.yaml new file mode 100644 index 00000000000..78f685d208e --- /dev/null +++ b/examples/metrics/docker/collector-config.yaml @@ -0,0 +1,18 @@ +receivers: + opencensus: + endpoint: "0.0.0.0:55678" + +exporters: + prometheus: + endpoint: "0.0.0.0:8889" + logging: {} + +processors: + batch: + queued_retry: + +service: + pipelines: + metrics: + receivers: [opencensus] + exporters: [logging, prometheus] diff --git a/examples/metrics/docker/docker-compose.yaml b/examples/metrics/docker/docker-compose.yaml new file mode 100644 index 00000000000..d12a0a4f18c --- /dev/null +++ b/examples/metrics/docker/docker-compose.yaml @@ -0,0 +1,19 @@ +version: "2" +services: + + otel-collector: + image: omnition/opentelemetry-collector-contrib:latest + command: ["--config=/conf/collector-config.yaml", "--log-level=DEBUG"] + volumes: + - ./collector-config.yaml:/conf/collector-config.yaml + ports: + - "8889:8889" # Prometheus exporter metrics + - "55678:55678" # OpenCensus receiver + + prometheus: + container_name: prometheus + image: prom/prometheus:latest + volumes: + - ./prometheus.yaml:/etc/prometheus/prometheus.yml + ports: + - "9090:9090" diff --git a/examples/metrics/docker/prometheus.yaml b/examples/metrics/docker/prometheus.yaml new file mode 100644 index 00000000000..4eb23572314 --- /dev/null +++ b/examples/metrics/docker/prometheus.yaml @@ -0,0 +1,5 @@ +scrape_configs: + - job_name: 'otel-collector' + scrape_interval: 5s + static_configs: + - targets: ['otel-collector:8889'] diff --git a/ext/opentelemetry-ext-otcollector/README.rst b/ext/opentelemetry-ext-otcollector/README.rst index 33d8d587479..200ec9a91d0 100644 --- a/ext/opentelemetry-ext-otcollector/README.rst +++ b/ext/opentelemetry-ext-otcollector/README.rst @@ -6,7 +6,7 @@ OpenTelemetry Collector Exporter .. |pypi| image:: https://badge.fury.io/py/opentelemetry-ext-otcollector.svg :target: https://pypi.org/project/opentelemetry-ext-otcollector/ -This library allows to export data to `OpenTelemetry Collector `_. +This library allows to export data to `OpenTelemetry Collector `_ , currently using OpenCensus receiver in Collector side. Installation ------------ @@ -16,8 +16,8 @@ Installation pip install opentelemetry-ext-otcollector -Usage ------ +Traces Usage +------------ The **OpenTelemetry Collector Exporter** allows to export `OpenTelemetry`_ traces to `OpenTelemetry Collector`_. @@ -48,6 +48,49 @@ The **OpenTelemetry Collector Exporter** allows to export `OpenTelemetry`_ trace with tracer.start_as_current_span("foo"): print("Hello world!") +Metrics Usage +------------- + +The **OpenTelemetry Collector Exporter** allows to export `OpenTelemetry`_ metrics to `OpenTelemetry Collector`_. + +.. code:: python + + from opentelemetry import metrics + from opentelemetry.ext.otcollector.metrics_exporter import CollectorMetricsExporter + from opentelemetry.sdk.metrics import Counter, MeterProvider + from opentelemetry.sdk.metrics.export.controller import PushController + + + # create a CollectorMetricsExporter + collector_exporter = CollectorMetricsExporter( + # optional: + # endpoint="myCollectorUrl:55678", + # service_name="test_service", + # host_name="machine/container name", + ) + + # Meter is responsible for creating and recording metrics + metrics.set_preferred_meter_provider_implementation(lambda _: MeterProvider()) + meter = metrics.get_meter(__name__) + # controller collects metrics created from meter and exports it via the + # exporter every interval + controller = PushController(meter, collector_exporter, 5) + counter = meter.create_metric( + "requests", + "number of requests", + "requests", + int, + Counter, + ("environment",), + ) + # Labelsets are used to identify key-values that are associated with a specific + # metric that you want to record. These are useful for pre-aggregation and can + # be used to store custom dimensions pertaining to a metric + label_set = meter.get_label_set({"environment": "staging"}) + + counter.add(25, label_set) + + References ---------- diff --git a/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py b/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py new file mode 100644 index 00000000000..12715035c25 --- /dev/null +++ b/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py @@ -0,0 +1,165 @@ +# Copyright 2020, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""OpenTelemetry Collector Metrics Exporter.""" + +import logging +from typing import Sequence + +import grpc +from opencensus.proto.agent.metrics.v1 import ( + metrics_service_pb2, + metrics_service_pb2_grpc, +) +from opencensus.proto.metrics.v1 import metrics_pb2 + +import opentelemetry.ext.otcollector.util as utils +from opentelemetry.sdk.metrics import Counter, Metric +from opentelemetry.sdk.metrics.export import ( + MetricRecord, + MetricsExporter, + MetricsExportResult, + aggregate, +) + +DEFAULT_ENDPOINT = "localhost:55678" + +logger = logging.getLogger(__name__) + + +# pylint: disable=no-member +class CollectorMetricsExporter(MetricsExporter): + """OpenTelemetry Collector metrics exporter. + + Args: + endpoint: OpenTelemetry Collector OpenCensus receiver endpoint. + service_name: Name of Collector service. + host_name: Host name. + client: MetricsService client stub. + """ + + def __init__( + self, + endpoint: str = DEFAULT_ENDPOINT, + service_name: str = None, + host_name: str = None, + client: metrics_service_pb2_grpc.MetricsServiceStub = None, + ): + self.endpoint = endpoint + if client is None: + channel = grpc.insecure_channel(self.endpoint) + self.client = metrics_service_pb2_grpc.MetricsServiceStub( + channel=channel + ) + else: + self.client = client + + self.node = utils.get_node(service_name, host_name) + + def export( + self, metric_records: Sequence[MetricRecord] + ) -> MetricsExportResult: + try: + responses = self.client.Export( + self.generate_metrics_requests(metric_records) + ) + + # Read response + for _ in responses: + pass + + except grpc.RpcError: + return MetricsExportResult.FAILED_RETRYABLE + + return MetricsExportResult.SUCCESS + + def shutdown(self) -> None: + pass + + def generate_metrics_requests( + self, metrics: Sequence[MetricRecord] + ) -> metrics_service_pb2.ExportMetricsServiceRequest: + collector_metrics = translate_to_collector(metrics) + service_request = metrics_service_pb2.ExportMetricsServiceRequest( + node=self.node, metrics=collector_metrics + ) + yield service_request + + +# pylint: disable=too-many-branches +def translate_to_collector( + metric_records: Sequence[MetricRecord], +) -> Sequence[metrics_pb2.Metric]: + collector_metrics = [] + for metric_record in metric_records: + + label_values = [] + label_keys = [] + for label_tuple in metric_record.label_set.labels: + label_keys.append(metrics_pb2.LabelKey(key=label_tuple[0])) + label_values.append( + metrics_pb2.LabelValue( + has_value=label_tuple[1] is not None, value=label_tuple[1] + ) + ) + + metric_descriptor = metrics_pb2.MetricDescriptor( + name=metric_record.metric.name, + description=metric_record.metric.description, + unit=metric_record.metric.unit, + type=get_collector_metric_type(metric_record.metric), + label_keys=label_keys, + ) + + timeseries = metrics_pb2.TimeSeries( + label_values=label_values, + points=[get_collector_point(metric_record)], + ) + collector_metrics.append( + metrics_pb2.Metric( + metric_descriptor=metric_descriptor, timeseries=[timeseries] + ) + ) + return collector_metrics + + +# pylint: disable=no-else-return +def get_collector_metric_type(metric: Metric) -> metrics_pb2.MetricDescriptor: + if isinstance(metric, Counter): + if metric.value_type == int: + return metrics_pb2.MetricDescriptor.CUMULATIVE_INT64 + elif metric.value_type == float: + return metrics_pb2.MetricDescriptor.CUMULATIVE_DOUBLE + return metrics_pb2.MetricDescriptor.UNSPECIFIED + + +def get_collector_point(metric_record: MetricRecord) -> metrics_pb2.Point: + point = metrics_pb2.Point( + timestamp=utils.proto_timestamp_from_time_ns( + metric_record.metric.bind( + metric_record.label_set + ).last_update_timestamp + ) + ) + if metric_record.metric.value_type == int: + point.int64_value = metric_record.aggregator.checkpoint + elif metric_record.metric.value_type == float: + point.double_value = metric_record.aggregator.checkpoint + else: + raise TypeError( + "Unsupported metric type: {}".format( + metric_record.metric.value_type + ) + ) + return point diff --git a/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py new file mode 100644 index 00000000000..f58f9768d60 --- /dev/null +++ b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py @@ -0,0 +1,202 @@ +# Copyright 2020, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from unittest import mock + +import grpc +from google.protobuf.timestamp_pb2 import Timestamp +from opencensus.proto.metrics.v1 import metrics_pb2 + +from opentelemetry import metrics +from opentelemetry.ext.otcollector import metrics_exporter +from opentelemetry.sdk.metrics import Counter, Measure, MeterProvider +from opentelemetry.sdk.metrics.export import ( + MetricRecord, + MetricsExportResult, + aggregate, +) + + +# pylint: disable=no-member +class TestCollectorMetricsExporter(unittest.TestCase): + @classmethod + def setUpClass(cls): + # pylint: disable=protected-access + cls._meter_defaults = ( + metrics._METER_PROVIDER, + metrics._METER_PROVIDER_FACTORY, + ) + metrics.set_preferred_meter_provider_implementation( + lambda _: MeterProvider() + ) + cls._meter = metrics.get_meter(__name__) + kvp = {"environment": "staging"} + cls._test_label_set = cls._meter.get_label_set(kvp) + + @classmethod + def tearDownClass(cls): + # pylint: disable=protected-access + ( + metrics._METER_PROVIDER, + metrics._METER_PROVIDER_FACTORY, + ) = cls._meter_defaults + + def test_constructor(self): + mock_get_node = mock.Mock() + patch = mock.patch( + "opentelemetry.ext.otcollector.util.get_node", + side_effect=mock_get_node, + ) + service_name = "testServiceName" + host_name = "testHostName" + client = grpc.insecure_channel("") + endpoint = "testEndpoint" + with patch: + exporter = metrics_exporter.CollectorMetricsExporter( + service_name=service_name, + host_name=host_name, + endpoint=endpoint, + client=client, + ) + + self.assertIs(exporter.client, client) + self.assertEqual(exporter.endpoint, endpoint) + mock_get_node.assert_called_with(service_name, host_name) + + def test_get_collector_metric_type(self): + result = metrics_exporter.get_collector_metric_type( + Counter("testName", "testDescription", "unit", int, None) + ) + self.assertIs(result, metrics_pb2.MetricDescriptor.CUMULATIVE_INT64) + result = metrics_exporter.get_collector_metric_type( + Counter("testName", "testDescription", "unit", float, None) + ) + self.assertIs(result, metrics_pb2.MetricDescriptor.CUMULATIVE_DOUBLE) + result = metrics_exporter.get_collector_metric_type( + Measure("testName", "testDescription", "unit", None, None) + ) + self.assertIs(result, metrics_pb2.MetricDescriptor.UNSPECIFIED) + + def test_get_collector_point(self): + aggregator = aggregate.CounterAggregator() + label_set = self._meter.get_label_set({"environment": "staging"}) + int_counter = self._meter.create_metric( + "testName", "testDescription", "unit", int, Counter + ) + float_counter = self._meter.create_metric( + "testName", "testDescription", "unit", float, Counter + ) + measure = self._meter.create_metric( + "testName", "testDescription", "unit", float, Measure + ) + result = metrics_exporter.get_collector_point( + MetricRecord(aggregator, label_set, int_counter) + ) + self.assertIsInstance(result, metrics_pb2.Point) + self.assertIsInstance(result.timestamp, Timestamp) + self.assertEqual(result.int64_value, 0) + aggregator.update(123.5) + aggregator.take_checkpoint() + result = metrics_exporter.get_collector_point( + MetricRecord(aggregator, label_set, float_counter) + ) + self.assertEqual(result.double_value, 123.5) + self.assertRaises( + TypeError, + metrics_exporter.get_collector_point( + MetricRecord(aggregator, label_set, measure) + ), + ) + + def test_export(self): + mock_client = mock.MagicMock() + mock_export = mock.MagicMock() + mock_client.Export = mock_export + host_name = "testHostName" + collector_exporter = metrics_exporter.CollectorMetricsExporter( + client=mock_client, host_name=host_name + ) + test_metric = self._meter.create_metric( + "testname", "testdesc", "unit", int, Counter, ["environment"] + ) + record = MetricRecord( + aggregate.CounterAggregator(), self._test_label_set, test_metric + ) + + result = collector_exporter.export([record]) + self.assertIs(result, MetricsExportResult.SUCCESS) + # pylint: disable=unsubscriptable-object + export_arg = mock_export.call_args[0] + service_request = next(export_arg[0]) + output_metrics = getattr(service_request, "metrics") + output_node = getattr(service_request, "node") + self.assertEqual(len(output_metrics), 1) + self.assertIsNotNone(getattr(output_node, "library_info")) + self.assertIsNotNone(getattr(output_node, "service_info")) + output_identifier = getattr(output_node, "identifier") + self.assertEqual( + getattr(output_identifier, "host_name"), "testHostName" + ) + + def test_translate_to_collector(self): + + test_metric = self._meter.create_metric( + "testname", "testdesc", "unit", int, Counter, ["environment"] + ) + aggregator = aggregate.CounterAggregator() + aggregator.update(123) + aggregator.take_checkpoint() + record = MetricRecord(aggregator, self._test_label_set, test_metric) + output_metrics = metrics_exporter.translate_to_collector([record]) + self.assertEqual(len(output_metrics), 1) + self.assertIsInstance(output_metrics[0], metrics_pb2.Metric) + self.assertEqual(output_metrics[0].metric_descriptor.name, "testname") + self.assertEqual( + output_metrics[0].metric_descriptor.description, "testdesc" + ) + self.assertEqual(output_metrics[0].metric_descriptor.unit, "unit") + self.assertEqual( + output_metrics[0].metric_descriptor.type, + metrics_pb2.MetricDescriptor.CUMULATIVE_INT64, + ) + self.assertEqual( + len(output_metrics[0].metric_descriptor.label_keys), 1 + ) + self.assertEqual( + output_metrics[0].metric_descriptor.label_keys[0].key, + "environment", + ) + self.assertEqual(len(output_metrics[0].timeseries), 1) + self.assertEqual(len(output_metrics[0].timeseries[0].label_values), 1) + self.assertEqual( + output_metrics[0].timeseries[0].label_values[0].has_value, True + ) + self.assertEqual( + output_metrics[0].timeseries[0].label_values[0].value, "staging" + ) + self.assertEqual(len(output_metrics[0].timeseries[0].points), 1) + self.assertEqual( + output_metrics[0].timeseries[0].points[0].timestamp.seconds, + record.metric.bind(record.label_set).last_update_timestamp + // 1000000000, + ) + self.assertEqual( + output_metrics[0].timeseries[0].points[0].timestamp.nanos, + record.metric.bind(record.label_set).last_update_timestamp + % 1000000000, + ) + self.assertEqual( + output_metrics[0].timeseries[0].points[0].int64_value, 123 + ) diff --git a/ext/opentelemetry-ext-otcollector/tests/test_otcollector_exporter.py b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_trace_exporter.py similarity index 100% rename from ext/opentelemetry-ext-otcollector/tests/test_otcollector_exporter.py rename to ext/opentelemetry-ext-otcollector/tests/test_otcollector_trace_exporter.py