Add unique identifier to Cloud Monitoring exporter (open-telemetry#841)

Andrew Xue · web-flow · commit d35e6268ada5 · 2020-06-22T22:00:49.000-07:00
diff --git a/docs/examples/cloud_monitoring/README.rst b/docs/examples/cloud_monitoring/README.rst
@@ -33,3 +33,16 @@ After running the example:
     * Go to the `Cloud Monitoring Metrics Explorer page <https://console.cloud.google.com/monitoring/metrics-explorer>`_.
     * In "Find resource type and metric" enter "OpenTelemetry/request_counter".
     * You can filter by labels and change the graphical output here as well.
+
+Troubleshooting
+--------------------------
+
+``One or more points were written more frequently than the maximum sampling period configured for the metric``
+##############################################################################################################
+
+Currently, Cloud Monitoring allows one write every 10 seconds for any unique tuple (metric_name, metric_label_value_1, metric_label_value_2, ...). The exporter should rate limit on its own but issues arise if:
+
+    * You are restarting the server more than once every 10 seconds.
+    * You have a multiple exporters (possibly on different threads) writing to the same tuple.
+
+For both cases, you can pass ``add_unique_identifier=True`` to the CloudMonitoringMetricsExporter constructor. This adds a UUID label_value, making the tuple unique again. For the first case, you can also choose to just wait longer than 10 seconds between restarts.
diff --git a/ext/opentelemetry-exporter-cloud-monitoring/src/opentelemetry/exporter/cloud_monitoring/__init__.py b/ext/opentelemetry-exporter-cloud-monitoring/src/opentelemetry/exporter/cloud_monitoring/__init__.py
@@ -1,4 +1,5 @@
 import logging
+import random
 from typing import Optional, Sequence
 
 import google.auth
@@ -17,14 +18,30 @@
 logger = logging.getLogger(__name__)
 MAX_BATCH_WRITE = 200
 WRITE_INTERVAL = 10
+UNIQUE_IDENTIFIER_KEY = "opentelemetry_id"
 
 
 # pylint is unable to resolve members of protobuf objects
 # pylint: disable=no-member
 class CloudMonitoringMetricsExporter(MetricsExporter):
-    """ Implementation of Metrics Exporter to Google Cloud Monitoring"""
-
-    def __init__(self, project_id=None, client=None):
+    """ Implementation of Metrics Exporter to Google Cloud Monitoring
+
+        You can manually pass in project_id and client, or else the
+        Exporter will take that information from Application Default
+        Credentials.
+
+    Args:
+        project_id: project id of your Google Cloud project.
+        client: Client to upload metrics to Google Cloud Monitoring.
+        add_unique_identifier: Add an identifier to each exporter metric. This
+            must be used when there exist two (or more) exporters that may
+            export to the same metric name within WRITE_INTERVAL seconds of
+            each other.
+    """
+
+    def __init__(
+        self, project_id=None, client=None, add_unique_identifier=False
+    ):
         self.client = client or MetricServiceClient()
         if not project_id:
             _, self.project_id = google.auth.default()
@@ -33,6 +50,11 @@ def __init__(self, project_id=None, client=None):
         self.project_name = self.client.project_path(self.project_id)
         self._metric_descriptors = {}
         self._last_updated = {}
+        self.unique_identifier = None
+        if add_unique_identifier:
+            self.unique_identifier = "{:08x}".format(
+                random.randint(0, 16 ** 8)
+            )
 
     def _add_resource_info(self, series: TimeSeries) -> None:
         """Add Google resource specific information (e.g. instance id, region).
@@ -97,6 +119,12 @@ def _get_metric_descriptor(
                 logger.warning(
                     "Label value %s is not a string, bool or integer", value
                 )
+
+        if self.unique_identifier:
+            descriptor["labels"].append(
+                LabelDescriptor(key=UNIQUE_IDENTIFIER_KEY, value_type="STRING")
+            )
+
         if isinstance(record.aggregator, SumAggregator):
             descriptor["metric_kind"] = MetricDescriptor.MetricKind.GAUGE
         else:
@@ -141,6 +169,11 @@ def export(
             for key, value in record.labels:
                 series.metric.labels[key] = str(value)
 
+            if self.unique_identifier:
+                series.metric.labels[
+                    UNIQUE_IDENTIFIER_KEY
+                ] = self.unique_identifier
+
             point = series.points.add()
             if instrument.value_type == int:
                 point.value.int64_value = record.aggregator.checkpoint
diff --git a/ext/opentelemetry-exporter-cloud-monitoring/tests/test_cloud_monitoring.py b/ext/opentelemetry-exporter-cloud-monitoring/tests/test_cloud_monitoring.py
@@ -21,6 +21,7 @@
 
 from opentelemetry.exporter.cloud_monitoring import (
     MAX_BATCH_WRITE,
+    UNIQUE_IDENTIFIER_KEY,
     WRITE_INTERVAL,
     CloudMonitoringMetricsExporter,
 )
@@ -289,3 +290,55 @@ def test_export(self):
                 mock.call(self.project_name, [series3]),
             ]
         )
+
+    def test_unique_identifier(self):
+        client = mock.Mock()
+        exporter1 = CloudMonitoringMetricsExporter(
+            project_id=self.project_id,
+            client=client,
+            add_unique_identifier=True,
+        )
+        exporter2 = CloudMonitoringMetricsExporter(
+            project_id=self.project_id,
+            client=client,
+            add_unique_identifier=True,
+        )
+        exporter1.project_name = self.project_name
+        exporter2.project_name = self.project_name
+
+        client.create_metric_descriptor.return_value = MetricDescriptor(
+            **{
+                "name": None,
+                "type": "custom.googleapis.com/OpenTelemetry/name",
+                "display_name": "name",
+                "description": "description",
+                "labels": [
+                    LabelDescriptor(
+                        key=UNIQUE_IDENTIFIER_KEY, value_type="STRING"
+                    ),
+                ],
+                "metric_kind": "GAUGE",
+                "value_type": "DOUBLE",
+            }
+        )
+
+        sum_agg_one = SumAggregator()
+        sum_agg_one.update(1)
+        metric_record = MetricRecord(MockMetric(), (), sum_agg_one,)
+        exporter1.export([metric_record])
+        exporter2.export([metric_record])
+
+        (
+            first_call,
+            second_call,
+        ) = client.create_metric_descriptor.call_args_list
+        self.assertEqual(first_call[0][1].labels[0].key, UNIQUE_IDENTIFIER_KEY)
+        self.assertEqual(
+            second_call[0][1].labels[0].key, UNIQUE_IDENTIFIER_KEY
+        )
+
+        first_call, second_call = client.create_time_series.call_args_list
+        self.assertNotEqual(
+            first_call[0][1][0].metric.labels[UNIQUE_IDENTIFIER_KEY],
+            second_call[0][1][0].metric.labels[UNIQUE_IDENTIFIER_KEY],
+        )
