Add periodic exporting metric reader (open-telemetry#2340)

srikanthccv · aabmass · web-flow · commit 83e6e17a4756 · 2022-01-21T11:42:53.000-05:00
* Add periodic exporting metric reader

* Fix lint

* Fix typing and env

* lint: black format

* formatting

* lot of changes

* Fix lint

* Lint again

* Pylint fix

* Review suggestions

* fix failures

* Address review suggestions

* Once wrapper, set callback, refactor, and more

* Apply suggestions from code review

Co-authored-by: Aaron Abbott &lt;aaronabbott@google.com&gt;

* Add preferred_temporality to exporter; update doc string

* Fix lint

Co-authored-by: Aaron Abbott &lt;aaronabbott@google.com&gt;
diff --git a/opentelemetry-sdk/setup.cfg b/opentelemetry-sdk/setup.cfg
@@ -47,6 +47,7 @@ install_requires =
     opentelemetry-semantic-conventions == 0.27b0
     setuptools >= 16.0
     dataclasses == 0.8; python_version < '3.7'
+    typing-extensions >= 3.7.4
 
 [options.packages.find]
 where = src
diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/export/__init__.py
@@ -12,13 +12,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
+import os
 from abc import ABC, abstractmethod
 from enum import Enum
-from os import linesep
+from os import environ, linesep
 from sys import stdout
-from typing import IO, Callable, Sequence
+from threading import Event, Thread
+from typing import IO, Callable, Iterable, Optional, Sequence
 
-from opentelemetry.sdk._metrics.point import Metric
+from opentelemetry.context import (
+    _SUPPRESS_INSTRUMENTATION_KEY,
+    attach,
+    detach,
+    set_value,
+)
+from opentelemetry.sdk._metrics.metric_reader import MetricReader
+from opentelemetry.sdk._metrics.point import AggregationTemporality, Metric
+from opentelemetry.util._once import Once
+
+_logger = logging.getLogger(__name__)
 
 
 class MetricExportResult(Enum):
@@ -33,6 +46,10 @@ class MetricExporter(ABC):
     in their own format.
     """
 
+    @property
+    def preferred_temporality(self) -> AggregationTemporality:
+        return AggregationTemporality.CUMULATIVE
+
     @abstractmethod
     def export(self, metrics: Sequence[Metric]) -> "MetricExportResult":
         """Exports a batch of telemetry data.
@@ -77,3 +94,82 @@ def export(self, metrics: Sequence[Metric]) -> MetricExportResult:
 
     def shutdown(self) -> None:
         pass
+
+
+class PeriodicExportingMetricReader(MetricReader):
+    """`PeriodicExportingMetricReader` is an implementation of `MetricReader`
+    that collects metrics based on a user-configurable time interval, and passes the
+    metrics to the configured exporter.
+    """
+
+    def __init__(
+        self,
+        exporter: MetricExporter,
+        export_interval_millis: Optional[float] = None,
+        export_timeout_millis: Optional[float] = None,
+    ) -> None:
+        super().__init__(preferred_temporality=exporter.preferred_temporality)
+        self._exporter = exporter
+        if export_interval_millis is None:
+            try:
+                export_interval_millis = float(
+                    environ.get("OTEL_METRIC_EXPORT_INTERVAL", 60000)
+                )
+            except ValueError:
+                _logger.warning(
+                    "Found invalid value for export interval, using default"
+                )
+                export_interval_millis = 60000
+        if export_timeout_millis is None:
+            try:
+                export_timeout_millis = float(
+                    environ.get("OTEL_METRIC_EXPORT_TIMEOUT", 30000)
+                )
+            except ValueError:
+                _logger.warning(
+                    "Found invalid value for export timeout, using default"
+                )
+                export_timeout_millis = 30000
+        self._export_interval_millis = export_interval_millis
+        self._export_timeout_millis = export_timeout_millis
+        self._shutdown = False
+        self._shutdown_event = Event()
+        self._shutdown_once = Once()
+        self._daemon_thread = Thread(target=self._ticker, daemon=True)
+        self._daemon_thread.start()
+        if hasattr(os, "register_at_fork"):
+            os.register_at_fork(
+                after_in_child=self._at_fork_reinit
+            )  # pylint: disable=protected-access
+
+    def _at_fork_reinit(self):
+        self._daemon_thread = Thread(target=self._ticker, daemon=True)
+        self._daemon_thread.start()
+
+    def _ticker(self) -> None:
+        interval_secs = self._export_interval_millis / 1e3
+        while not self._shutdown_event.wait(interval_secs):
+            self.collect()
+        # one last collection below before shutting down completely
+        self.collect()
+
+    def _receive_metrics(self, metrics: Iterable[Metric]) -> None:
+        token = attach(set_value(_SUPPRESS_INSTRUMENTATION_KEY, True))
+        try:
+            self._exporter.export(metrics)
+        except Exception as e:  # pylint: disable=broad-except,invalid-name
+            _logger.exception("Exception while exporting metrics %s", str(e))
+        detach(token)
+
+    def shutdown(self) -> bool:
+        def _shutdown():
+            self._shutdown = True
+
+        did_set = self._shutdown_once.do_once(_shutdown)
+        if not did_set:
+            _logger.warning("Can't shutdown multiple times")
+            return False
+
+        self._shutdown_event.set()
+        self._daemon_thread.join()
+        return self._exporter.shutdown()
diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/metric_reader.py b/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/metric_reader.py
@@ -12,6 +12,59 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
+from abc import ABC, abstractmethod
+from typing import Callable, Iterable
 
-class MetricReader:
-    pass
+from typing_extensions import final
+
+from opentelemetry.sdk._metrics.point import AggregationTemporality, Metric
+
+_logger = logging.getLogger(__name__)
+
+
+class MetricReader(ABC):
+    def __init__(
+        self,
+        preferred_temporality: AggregationTemporality = AggregationTemporality.CUMULATIVE,
+    ) -> None:
+        self._collect: Callable[
+            ["MetricReader", AggregationTemporality], Iterable[Metric]
+        ] = None
+        self._preferred_temporality = preferred_temporality
+
+    @final
+    def collect(self) -> None:
+        """Collects the metrics from the internal SDK state and
+        invokes the `_receive_metrics` with the collection.
+        """
+        if self._collect is None:
+            _logger.warning(
+                "Cannot call collect on a MetricReader until it is registered on a MeterProvider"
+            )
+            return
+        self._receive_metrics(self._collect(self, self._preferred_temporality))
+
+    @final
+    def _set_collect_callback(
+        self,
+        func: Callable[
+            ["MetricReader", AggregationTemporality], Iterable[Metric]
+        ],
+    ) -> None:
+        """This function is internal to the SDK. It should not be called or overriden by users"""
+        self._collect = func
+
+    @abstractmethod
+    def _receive_metrics(self, metrics: Iterable[Metric]):
+        """Called by `MetricReader.collect` when it receives a batch of metrics"""
+
+    @abstractmethod
+    def shutdown(self) -> bool:
+        """Shuts down the MetricReader. This method provides a way
+        for the MetricReader to do any cleanup required. A metric reader can
+        only be shutdown once, any subsequent calls are ignored and return
+        failure status.
+
+        When a `MetricReader` is registered on a `MeterProvider`, `MeterProvider.shutdown` will invoke this automatically.
+        """
diff --git a/opentelemetry-sdk/tests/metrics/test_periodic_exporting_metric_reader.py b/opentelemetry-sdk/tests/metrics/test_periodic_exporting_metric_reader.py
@@ -0,0 +1,126 @@
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+from unittest.mock import Mock
+
+from opentelemetry.sdk._metrics.export import (
+    MetricExporter,
+    PeriodicExportingMetricReader,
+)
+from opentelemetry.sdk._metrics.point import Gauge, Metric, Sum
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.test.concurrency_test import ConcurrencyTestBase
+from opentelemetry.util._time import _time_ns
+
+
+class FakeMetricsExporter(MetricExporter):
+    def __init__(self, wait=0):
+        self.wait = wait
+        self.metrics = []
+        self._shutdown = False
+
+    def export(self, metrics):
+        time.sleep(self.wait)
+        self.metrics.extend(metrics)
+        return True
+
+    def shutdown(self):
+        self._shutdown = True
+
+
+metrics_list = [
+    Metric(
+        name="sum_name",
+        attributes={},
+        description="",
+        instrumentation_info=None,
+        resource=Resource.create(),
+        unit="",
+        point=Sum(
+            start_time_unix_nano=_time_ns(),
+            time_unix_nano=_time_ns(),
+            value=2,
+            aggregation_temporality=1,
+            is_monotonic=True,
+        ),
+    ),
+    Metric(
+        name="gauge_name",
+        attributes={},
+        description="",
+        instrumentation_info=None,
+        resource=Resource.create(),
+        unit="",
+        point=Gauge(
+            time_unix_nano=_time_ns(),
+            value=2,
+        ),
+    ),
+]
+
+
+class TestPeriodicExportingMetricReader(ConcurrencyTestBase):
+    def test_defaults(self):
+        pmr = PeriodicExportingMetricReader(FakeMetricsExporter())
+        self.assertEqual(pmr._export_interval_millis, 60000)
+        self.assertEqual(pmr._export_timeout_millis, 30000)
+        pmr.shutdown()
+
+    def _create_periodic_reader(
+        self, metrics, exporter, collect_wait=0, interval=60000
+    ):
+
+        pmr = PeriodicExportingMetricReader(exporter, interval)
+
+        def _collect(reader, temp):
+            time.sleep(collect_wait)
+            pmr._receive_metrics(metrics)
+
+        pmr._set_collect_callback(_collect)
+        return pmr
+
+    def test_ticker_called(self):
+        collect_mock = Mock()
+        pmr = PeriodicExportingMetricReader(Mock(), 1)
+        pmr._set_collect_callback(collect_mock)
+        time.sleep(0.1)
+        self.assertTrue(collect_mock.assert_called_once)
+        pmr.shutdown()
+
+    def test_ticker_collects_metrics(self):
+        exporter = FakeMetricsExporter()
+
+        pmr = self._create_periodic_reader(
+            metrics_list, exporter, interval=100
+        )
+        time.sleep(0.2)
+        self.assertEqual(exporter.metrics, metrics_list)
+        pmr.shutdown()
+
+    def test_shutdown(self):
+        exporter = FakeMetricsExporter()
+
+        pmr = self._create_periodic_reader([], exporter)
+        pmr.shutdown()
+        self.assertEqual(exporter.metrics, [])
+        self.assertTrue(pmr._shutdown)
+        self.assertTrue(exporter._shutdown)
+
+    def test_shutdown_multiple_times(self):
+        pmr = self._create_periodic_reader([], Mock())
+        with self.assertLogs(level="WARNING") as w:
+            self.run_with_many_threads(pmr.shutdown)
+            self.assertTrue("Can't shutdown multiple times", w.output[0])
+        pmr.shutdown()
