From 60fe4a4a2498f7829f58c6794b237c0da011e357 Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Wed, 15 Apr 2020 16:51:00 -0700 Subject: [PATCH 01/10] Adding live metrics capabilities --- .../auto_collection/live_metrics/__init__.py | 151 ++++++++++++++++ .../auto_collection/live_metrics/exporter.py | 163 ++++++++++++++++++ .../auto_collection/live_metrics/manager.py | 148 ++++++++++++++++ .../auto_collection/live_metrics/sender.py | 47 +++++ .../auto_collection/live_metrics/__init__.py | 2 + .../live_metrics/test_exporter.py | 81 +++++++++ .../live_metrics/test_manager.py | 39 +++++ .../live_metrics/test_sender.py | 85 +++++++++ 8 files changed, 716 insertions(+) create mode 100644 azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py create mode 100644 azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py create mode 100644 azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py create mode 100644 azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py create mode 100644 azure_monitor/tests/auto_collection/live_metrics/__init__.py create mode 100644 azure_monitor/tests/auto_collection/live_metrics/test_exporter.py create mode 100644 azure_monitor/tests/auto_collection/live_metrics/test_manager.py create mode 100644 azure_monitor/tests/auto_collection/live_metrics/test_sender.py diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py new file mode 100644 index 0000000..a8e7887 --- /dev/null +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py @@ -0,0 +1,151 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# +import time +import typing +import uuid + +from azure_monitor.protocol import BaseObject +from azure_monitor.utils import azure_monitor_context + +LIVE_METRICS_SUBSCRIBED_HEADER = "x-ms-qps-subscribed" +STREAM_ID = str(uuid.uuid4()) + + +def create_metric_envelope(instrumentation_key: str): + envelope = LiveMetricEnvelope( + documents=None, + instance=azure_monitor_context.get("ai.cloud.roleInstance"), + instrumentation_key=instrumentation_key, + invariant_version=1, # 1 -> v1 QPS protocol, + machine_name=azure_monitor_context.get("ai.device.id"), + metrics=None, + stream_id=STREAM_ID, + timestamp="/Date({0})/".format(str(int(time.time()) * 1000)), + version=azure_monitor_context.get("ai.internal.sdkVersion"), + ) + return envelope + + +class LiveMetricDocumentProperty(BaseObject): + + __slots__ = ("key", "value") + + def __init__(self, key: str, value: str) -> None: + self.key = key + self.value = value + + +class LiveMetricDocument(BaseObject): + + __slots__ = ( + "__type", + "document_type", + "version", + "operation_id", + "properties", + ) + + def __init__( + self, + __type: str = "", + document_type: str = "", + version: str = "", + operation_id: str = "", + properties: typing.List[LiveMetricDocumentProperty] = None, + ) -> None: + self.__type = __type + self.document_type = document_type + self.version = version + self.operation_id = operation_id + self.properties = properties + + def to_dict(self): + return { + "__type": self.__type, + "DocumentType": self.document_type, + "Version": self.version, + "OperationId": self.operation_id, + "Properties": self.properties.to_dict() + if self.properties + else None, + } + + +class LiveMetric(BaseObject): + + __slots__ = ("name", "value", "weight") + + def __init__(self, name: str, value: str, weight: int) -> None: + self.name = name + self.value = value + self.weight = weight + + def to_dict(self): + return {"Name": self.name, "Value": self.value, "Weight": self.weight} + + +class LiveMetricEnvelope(BaseObject): + """Envelope to send data to Live Metrics service. + + Args: + documents: An array of detailed failure documents, each representing a full SDK telemetry item. + instance: Instance name. Either a cloud RoleInstance (if running in Azure), or the machine name otherwise. + instrumentation_key: Instrumentation key that the agent is instrumented with. While SDK can send to multiple ikeys, + it must select a single ikey to send QuickPulse data to - and only consider telemetry items sent to that ikey while collecting. + invariant_version: Version of QPS protocol that SDK supports. Currently, the latest is 2. + machine_name: Machine name. + metrics: Metrics + stream_id: A random GUID generated at start-up. Must remain unchanged while the application is running. + timestamp: UTC time the request was made in the Microsoft JSON format, e.g. "/Date(1478555534692)/". + version: SDK version (not specific to QuickPulse). Please make sure that there's some sort of prefix to identify the client (.NET SDK, Java SDK, etc.). + """ + + __slots__ = ( + "documents", + "instance", + "instrumentation_key", + "invariant_version", + "machine_name", + "metrics", + "stream_id", + "timestamp", + "version", + ) + + def __init__( + self, + documents: typing.List[LiveMetricDocument] = None, + instance: str = "", + instrumentation_key: str = "", + invariant_version: int = 1, + machine_name: str = "", + metrics: typing.List[LiveMetric] = None, + stream_id: str = "", + timestamp: str = "", + version: str = "", + ) -> None: + if metrics is None: + metrics = [] + self.documents = documents + self.instance = instance + self.instrumentation_key = instrumentation_key + self.invariant_version = invariant_version + self.machine_name = machine_name + self.metrics = metrics + self.stream_id = stream_id + self.timestamp = timestamp + self.version = version + + def to_dict(self): + return { + "Documents": self.documents.to_dict() if self.documents else None, + "Instance": self.instance, + "InstrumentationKey": self.instrumentation_key, + "InvariantVersion": self.invariant_version, + "MachineName": self.machine_name, + "Metrics": list(map(lambda x: x.to_dict(), self.metrics)), + "StreamId": self.stream_id, + "Timestamp": self.timestamp, + "Version": self.version, + } diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py new file mode 100644 index 0000000..3f7674f --- /dev/null +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py @@ -0,0 +1,163 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# +import collections +import logging +import typing + +from opentelemetry.sdk.metrics import Counter, Observer +from opentelemetry.sdk.metrics.export import ( + MetricRecord, + MetricsExporter, + MetricsExportResult, +) + +from azure_monitor.protocol import Envelope +from azure_monitor.sdk.auto_collection import live_metrics +from azure_monitor.sdk.auto_collection.live_metrics import ( + LiveMetric, + LiveMetricDocument, + LiveMetricDocumentProperty, + LiveMetricEnvelope, +) +from azure_monitor.sdk.auto_collection.live_metrics.sender import ( + LiveMetricsSender, +) + +logger = logging.getLogger(__name__) + + +class LiveMetricsExporter(MetricsExporter): + def __init__(self, instrumentation_key): + self._instrumentation_key = instrumentation_key + self._sender = LiveMetricsSender(self._instrumentation_key) + self._subscribed = True + self._document_envelopes = collections.deque() + + def add_document(self, envelope: Envelope): + self._document_envelopes.append(envelope) + + def export( + self, metric_records: typing.Sequence[MetricRecord] + ) -> MetricsExportResult: + envelope = self._metric_to_live_metrics_envelope(metric_records) + try: + response = self._sender.post(envelope) + if response.ok: + self._subscribed = ( + response.headers.get( + live_metrics.LIVE_METRICS_SUBSCRIBED_HEADER + ) + == "true" + ) + return MetricsExportResult.SUCCESS + + except Exception: # pylint: disable=broad-except + logger.exception("Exception occurred while exporting the data.") + + return MetricsExportResult.FAILED_NOT_RETRYABLE + + def _metric_to_live_metrics_envelope( + self, metric_records: typing.Sequence[MetricRecord] + ) -> LiveMetricEnvelope: + + envelope = live_metrics.create_metric_envelope( + self._instrumentation_key + ) + envelope.documents = self._get_live_metric_documents() + envelope.metrics = [] + + # Add metrics + for metric_record in metric_records: + value = 0 + metric = metric_record.metric + if isinstance(metric, Counter): + value = metric_record.aggregator.checkpoint + elif isinstance(metric, Observer): + value = metric_record.aggregator.checkpoint.last + if not value: + value = 0 + envelope.metrics.append( + LiveMetric(name=metric.name, value=value, weight=1) + ) + + return envelope + + def _get_live_metric_documents( + self, + ) -> typing.Sequence[LiveMetricDocument]: + live_metric_documents = [] + while self._document_envelopes: + for envelope in self._document_envelopes.popleft(): + base_type = envelope.data.baseType + if base_type: + document = LiveMetricDocument( + __type=self._get_live_metric_type(base_type), + document_type=self._get_live_metric_document_type( + base_type + ), + properties=self._get_aggregated_properties(envelope), + version="1.0", + ) + live_metric_documents.append(document) + else: + logger.warning( + "Document type invalid; not sending live metric document" + ) + + return live_metric_documents + + def _get_live_metric_type(self, base_type: str) -> str: + if base_type == "EventData": + return "EventTelemetryDocument" + elif base_type == "ExceptionData": + return "ExceptionTelemetryDocument" + elif base_type == "MessageData": + return "TraceTelemetryDocument" + elif base_type == "MetricData": + return "MetricTelemetryDocument" + elif base_type == "RequestData": + return "RequestTelemetryDocument" + elif base_type == "RemoteDependencyData": + return "DependencyTelemetryDocument" + elif base_type == "AvailabilityData": + return "AvailabilityTelemetryDocument" + + def _get_live_metric_document_type(self, base_type: str) -> str: + if base_type == "EventData": + return "Event" + elif base_type == "ExceptionData": + return "Exception" + elif base_type == "MessageData": + return "Trace" + elif base_type == "MetricData": + return "Metric" + elif base_type == "RequestData": + return "Request" + elif base_type == "RemoteDependencyData": + return "RemoteDependency" + elif base_type == "AvailabilityData": + return "Availability" + + def _get_aggregated_properties( + self, envelope: Envelope + ) -> typing.List[LiveMetricDocumentProperty]: + + aggregated_properties = [] + measurements = ( + envelope.data.base_data.measurements + if envelope.data.base_data.measurements + else [] + ) + for key in measurements: + prop = LiveMetricDocumentProperty(key=key, value=measurements[key]) + aggregated_properties.append(prop) + properties = ( + envelope.data.base_data.properties + if envelope.data.base_data.properties + else [] + ) + for key in properties: + prop = LiveMetricDocumentProperty(key=key, value=properties[key]) + aggregated_properties.append(prop) + return aggregated_properties diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py new file mode 100644 index 0000000..23cb419 --- /dev/null +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py @@ -0,0 +1,148 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# +import threading +import time + +from opentelemetry.sdk.metrics.export import MetricsExportResult + +from azure_monitor.sdk.auto_collection import live_metrics +from azure_monitor.sdk.auto_collection.live_metrics.exporter import ( + LiveMetricsExporter, +) +from azure_monitor.sdk.auto_collection.live_metrics.sender import ( + LiveMetricsSender, +) + +FALLBACK_INTERVAL = 60 +PING_INTERVAL = 5 +POST_INTERVAL = 1 +MAIN_INTERVAL = 2 + + +class LiveMetricsManager(threading.Thread): + + daemon = True + + def __init__(self, meter, instrumentation_key): + super().__init__() + self.thread_event = threading.Event() + self.interval = MAIN_INTERVAL + self._instrumentation_key = instrumentation_key + self._is_user_subscribed = False + self._meter = meter + self._post = None + self._ping = LiveMetricsPing(self._instrumentation_key) + self.start() + + def run(self): + self.check_if_user_is_subscribed() + while not self.thread_event.wait(self.interval): + self.check_if_user_is_subscribed() + + def check_if_user_is_subscribed(self): + if self._ping: + if self._ping.is_user_subscribed: + # Switch to Post + self._ping.shutdown() + self._ping = None + self._post = LiveMetricsPost( + self._meter, self._instrumentation_key + ) + if self._post: + if not self._post.is_user_subscribed: + # Switch to Ping + self._post.shutdown() + self._post = None + self._ping = LiveMetricsPing(self._instrumentation_key) + + def shutdown(self): + self.thread_event.set() + + +class LiveMetricsPing(threading.Thread): + + daemon = True + + def __init__(self, instrumentation_key): + super().__init__() + self.instrumentation_key = instrumentation_key + self.thread_event = threading.Event() + self.interval = PING_INTERVAL + self.is_user_subscribed = False + self.last_send_succeeded = False + self.last_request_success_time = 0 + self.sender = LiveMetricsSender(self.instrumentation_key) + self.start() + + def run(self): + self.ping() + while not self.thread_event.wait(self.interval): + self.ping() + + def ping(self): + envelope = live_metrics.create_metric_envelope( + self.instrumentation_key + ) + response = self.sender.ping(envelope) + if response.ok: + if not self.last_send_succeeded: + self.interval = PING_INTERVAL + self.last_send_succeeded = True + self.last_request_success_time = time.time() + if ( + response.headers.get( + live_metrics.LIVE_METRICS_SUBSCRIBED_HEADER + ) + == "true" + ): + self.is_user_subscribed = True + else: + self.last_send_succeeded = False + if time.time() >= self.last_request_success_time + 20: + self.interval = FALLBACK_INTERVAL + + def shutdown(self): + self.thread_event.set() + + +class LiveMetricsPost(threading.Thread): + + daemon = True + + def __init__(self, meter, instrumentation_key): + super().__init__() + self.instrumentation_key = instrumentation_key + self.meter = meter + self.thread_event = threading.Event() + self.interval = POST_INTERVAL + self.is_user_subscribed = True + self.last_send_succeeded = False + self.last_request_success_time = time.time() + self.exporter = LiveMetricsExporter(self.instrumentation_key) + self.start() + + def run(self): + self.post() + while not self.thread_event.wait(self.interval): + self.post() + + def post(self): + self.meter.collect() + result = self.exporter.export(self.meter.batcher.checkpoint_set()) + self.meter.batcher.finished_collection() + if result == MetricsExportResult.SUCCESS: + self.last_request_success_time = time.time() + if not self.last_send_succeeded: + self.interval = POST_INTERVAL + self.last_send_succeeded = True + + if self.exporter.subscribed: + self.is_user_subscribed = True + else: + self.last_send_succeeded = False + if time.time() >= self.last_request_success_time + 60: + self.interval = FALLBACK_INTERVAL + + def shutdown(self): + self.thread_event.set() diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py new file mode 100644 index 0000000..04df2d8 --- /dev/null +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py @@ -0,0 +1,47 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# +import json +import logging +import time + +import requests + +from azure_monitor.sdk.auto_collection.live_metrics import LiveMetricEnvelope + +DEFAULT_LIVEMETRICS_ENDPOINT = "https://rt.services.visualstudio.com" +LIVE_METRICS_TRANSMISSION_TIME_HEADER = "x-ms-qps-transmission-time" + +logger = logging.getLogger(__name__) + + +class LiveMetricsSender: + def __init__(self, instrumentation_key: str): + self._endpoint = DEFAULT_LIVEMETRICS_ENDPOINT + self._instrumentation_key = instrumentation_key + + def ping(self, envelope: LiveMetricEnvelope): + return self._send_request(json.dumps(envelope.to_dict()), "ping") + + def post(self, envelope: LiveMetricEnvelope): + return self._send_request(json.dumps([envelope.to_dict()]), "post") + + def _send_request(self, data: str, request_type: str) -> requests.Response: + try: + url = "{0}/QuickPulseService.svc/{1}?ikey={2}".format( + self._endpoint, request_type, self._instrumentation_key + ) + response = requests.post( + url=url, + data=data, + headers={ + "Expect": "100-continue", + "Content-Type": "application/json; charset=utf-8", + LIVE_METRICS_TRANSMISSION_TIME_HEADER: str( + round(time.time()) * 1000 + ), + }, + ) + except Exception as ex: + logger.warning("Failed to send live metrics: %s.", ex) + return response diff --git a/azure_monitor/tests/auto_collection/live_metrics/__init__.py b/azure_monitor/tests/auto_collection/live_metrics/__init__.py new file mode 100644 index 0000000..5b7f7a9 --- /dev/null +++ b/azure_monitor/tests/auto_collection/live_metrics/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py b/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py new file mode 100644 index 0000000..613f5e0 --- /dev/null +++ b/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py @@ -0,0 +1,81 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import unittest +from unittest import mock + +from opentelemetry import metrics +from opentelemetry.sdk.metrics import Counter, MeterProvider +from opentelemetry.sdk.metrics.export import MetricRecord, MetricsExportResult +from opentelemetry.sdk.metrics.export.aggregate import CounterAggregator +import requests + +from azure_monitor.protocol import Envelope +from azure_monitor.sdk.auto_collection.live_metrics.exporter import ( + LiveMetricsExporter, +) + + +# pylint: disable=protected-access +class TestLiveMetricsExporter(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls._instrumentation_key = "99c42f65-1656-4c41-afde-bd86b709a4a7" + metrics.set_meter_provider(MeterProvider()) + cls._meter = metrics.get_meter(__name__) + cls._test_metric = cls._meter.create_metric( + "testname", "testdesc", "unit", int, Counter, ["environment"] + ) + cls._test_labels = tuple({"environment": "staging"}.items()) + + def test_constructor(self): + """Test the constructor.""" + exporter = LiveMetricsExporter( + instrumentation_key=self._instrumentation_key + ) + self.assertEqual(exporter._subscribed, True) + self.assertEqual( + exporter._instrumentation_key, self._instrumentation_key + ) + + def test_add_document(self): + """Test adding a document.""" + exporter = LiveMetricsExporter( + instrumentation_key=self._instrumentation_key + ) + envelope = Envelope() + exporter.add_document(envelope) + self.assertEqual(exporter._document_envelopes.pop(), envelope) + + def test_export(self): + """Test export.""" + record = MetricRecord( + CounterAggregator(), self._test_labels, self._test_metric + ) + exporter = LiveMetricsExporter( + instrumentation_key=self._instrumentation_key + ) + with mock.patch( + "azure_monitor.sdk.auto_collection.live_metrics.sender.LiveMetricsSender.post" + ) as request: + response = requests.Response() + response.status_code = 200 + request.return_value = response + result = exporter.export([record]) + self.assertEqual(result, MetricsExportResult.SUCCESS) + + def test_export_failed(self): + record = MetricRecord( + CounterAggregator(), self._test_labels, self._test_metric + ) + exporter = LiveMetricsExporter( + instrumentation_key=self._instrumentation_key + ) + with mock.patch( + "azure_monitor.sdk.auto_collection.live_metrics.sender.LiveMetricsSender.post" + ) as request: + response = requests.Response() + response.status_code = 400 + request.return_value = response + result = exporter.export([record]) + self.assertEqual(result, MetricsExportResult.FAILED_NOT_RETRYABLE) diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_manager.py b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py new file mode 100644 index 0000000..f0291a4 --- /dev/null +++ b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py @@ -0,0 +1,39 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import time +import unittest +from unittest import mock + +from opentelemetry import metrics +from opentelemetry.sdk.metrics import Counter, Measure, MeterProvider + +from azure_monitor.sdk.auto_collection.live_metrics.manager import ( + LiveMetricsManager, +) + + +# pylint: disable=protected-access +class TestAutoCollection(unittest.TestCase): + @classmethod + def setUpClass(cls): + metrics.set_meter_provider(MeterProvider()) + cls._meter = metrics.get_meter(__name__) + cls._test_metric = cls._meter.create_metric( + "testname", "testdesc", "unit", int, Counter, ["environment"] + ) + testing_labels = {"environment": "testing"} + cls._test_metric.add(5, testing_labels) + cls._instrumentation_key = "99c42f65-1656-4c41-afde-bd86b709a4a7" + + @classmethod + def tearDownClass(cls): + metrics._METER_PROVIDER = None + + def test_constructor(self): + """Test the constructor.""" + # LiveMetricsManager( + # meter=self._meter, instrumentation_key=self._instrumentation_key + # ) + # time.sleep(200) + # self.assertEqual(True, False) diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_sender.py b/azure_monitor/tests/auto_collection/live_metrics/test_sender.py new file mode 100644 index 0000000..5fdead2 --- /dev/null +++ b/azure_monitor/tests/auto_collection/live_metrics/test_sender.py @@ -0,0 +1,85 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import unittest +from unittest import mock + +from azure_monitor.sdk.auto_collection.live_metrics import LiveMetricEnvelope +from azure_monitor.sdk.auto_collection.live_metrics.sender import ( + LiveMetricsSender, +) + + +# pylint: disable=protected-access +class TestLiveMetricsSender(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls._instrumentation_key = "99c42f65-1656-4c41-afde-bd86b709a4a7" + + def test_constructor(self): + """Test the constructor.""" + sender = LiveMetricsSender( + instrumentation_key=self._instrumentation_key + ) + self.assertEqual( + sender._endpoint, "https://rt.services.visualstudio.com" + ) + self.assertEqual( + sender._instrumentation_key, self._instrumentation_key + ) + + def test_ping(self): + """Test ping.""" + sender = LiveMetricsSender( + instrumentation_key=self._instrumentation_key + ) + envelope = LiveMetricEnvelope() + with mock.patch("requests.post") as request: + sender.ping(envelope) + self.assertTrue(request.called) + self.assertEqual( + request.call_args[1].get("url"), + "https://rt.services.visualstudio.com/QuickPulseService.svc/ping?ikey={0}".format( + self._instrumentation_key + ), + ) + self.assertEqual( + request.call_args[1].get("data"), + '{"Documents": null, "Instance": "", "InstrumentationKey": "", "InvariantVersion": 1, "MachineName": "", "Metrics": [], "StreamId": "", "Timestamp": "", "Version": ""}', + ) + headers = request.call_args[1].get("headers") + self.assertEqual(headers.get("Expect"), "100-continue") + self.assertEqual( + headers.get("Content-Type"), "application/json; charset=utf-8" + ) + self.assertTrue( + headers.get("x-ms-qps-transmission-time").isdigit() + ) + + def test_post(self): + """Test post.""" + sender = LiveMetricsSender( + instrumentation_key=self._instrumentation_key + ) + envelope = LiveMetricEnvelope() + with mock.patch("requests.post") as request: + sender.post(envelope) + self.assertTrue(request.called) + self.assertEqual( + request.call_args[1].get("url"), + "https://rt.services.visualstudio.com/QuickPulseService.svc/post?ikey={0}".format( + self._instrumentation_key + ), + ) + self.assertEqual( + request.call_args[1].get("data"), + '[{"Documents": null, "Instance": "", "InstrumentationKey": "", "InvariantVersion": 1, "MachineName": "", "Metrics": [], "StreamId": "", "Timestamp": "", "Version": ""}]', + ) + headers = request.call_args[1].get("headers") + self.assertEqual(headers.get("Expect"), "100-continue") + self.assertEqual( + headers.get("Content-Type"), "application/json; charset=utf-8" + ) + self.assertTrue( + headers.get("x-ms-qps-transmission-time").isdigit() + ) From c18e58932bc09aaf71f6064a24dea40136322cfa Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Thu, 16 Apr 2020 14:55:52 -0700 Subject: [PATCH 02/10] Fix shutdown --- .../auto_collection/live_metrics/manager.py | 4 ++++ .../live_metrics/test_manager.py | 22 ++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py index 23cb419..f7f2f6e 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py @@ -57,6 +57,10 @@ def check_if_user_is_subscribed(self): self._ping = LiveMetricsPing(self._instrumentation_key) def shutdown(self): + if self._ping: + self._ping.shutdown() + if self._post: + self._post.shutdown() self.thread_event.set() diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_manager.py b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py index f0291a4..966fac2 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_manager.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py @@ -14,7 +14,7 @@ # pylint: disable=protected-access -class TestAutoCollection(unittest.TestCase): +class TestLiveMetricsManager(unittest.TestCase): @classmethod def setUpClass(cls): metrics.set_meter_provider(MeterProvider()) @@ -25,15 +25,25 @@ def setUpClass(cls): testing_labels = {"environment": "testing"} cls._test_metric.add(5, testing_labels) cls._instrumentation_key = "99c42f65-1656-4c41-afde-bd86b709a4a7" + cls._manager = None @classmethod def tearDownClass(cls): metrics._METER_PROVIDER = None + def tearDown(self): + self._manager.shutdown() + def test_constructor(self): """Test the constructor.""" - # LiveMetricsManager( - # meter=self._meter, instrumentation_key=self._instrumentation_key - # ) - # time.sleep(200) - # self.assertEqual(True, False) + with mock.patch("requests.post"): + self._manager = LiveMetricsManager( + meter=self._meter, + instrumentation_key=self._instrumentation_key, + ) + self.assertFalse(self._manager._is_user_subscribed) + self.assertEqual( + self._manager._instrumentation_key, self._instrumentation_key + ) + self.assertEqual(self._manager._meter, self._meter) + From 09a623f0dbe7286584d9ab10d8a56082f336ac6e Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Tue, 21 Apr 2020 15:53:07 -0700 Subject: [PATCH 03/10] Addressing comments --- azure_monitor/src/azure_monitor/protocol.py | 124 +++++++++++++++++ .../auto_collection/live_metrics/__init__.py | 126 +----------------- .../auto_collection/live_metrics/exporter.py | 11 +- .../auto_collection/live_metrics/manager.py | 18 +++ .../auto_collection/live_metrics/sender.py | 7 +- .../live_metrics/test_exporter.py | 2 +- .../live_metrics/test_manager.py | 32 +++++ 7 files changed, 190 insertions(+), 130 deletions(-) diff --git a/azure_monitor/src/azure_monitor/protocol.py b/azure_monitor/src/azure_monitor/protocol.py index 9f68f16..e10c8d4 100644 --- a/azure_monitor/src/azure_monitor/protocol.py +++ b/azure_monitor/src/azure_monitor/protocol.py @@ -561,3 +561,127 @@ def to_dict(self): "properties": self.properties, "measurements": self.measurements, } + + +class LiveMetricDocumentProperty(BaseObject): + + __slots__ = ("key", "value") + + def __init__(self, key: str, value: str) -> None: + self.key = key + self.value = value + + +class LiveMetricDocument(BaseObject): + + __slots__ = ( + "__type", + "document_type", + "version", + "operation_id", + "properties", + ) + + def __init__( + self, + __type: str = "", + document_type: str = "", + version: str = "", + operation_id: str = "", + properties: typing.List[LiveMetricDocumentProperty] = None, + ) -> None: + self.__type = __type + self.document_type = document_type + self.version = version + self.operation_id = operation_id + self.properties = properties + + def to_dict(self): + return { + "__type": self.__type, + "DocumentType": self.document_type, + "Version": self.version, + "OperationId": self.operation_id, + "Properties": self.properties.to_dict() + if self.properties + else None, + } + + +class LiveMetric(BaseObject): + + __slots__ = ("name", "value", "weight") + + def __init__(self, name: str, value: str, weight: int) -> None: + self.name = name + self.value = value + self.weight = weight + + def to_dict(self): + return {"Name": self.name, "Value": self.value, "Weight": self.weight} + + +class LiveMetricEnvelope(BaseObject): + """Envelope to send data to Live Metrics service. + + Args: + documents: An array of detailed failure documents, each representing a full SDK telemetry item. + instance: Instance name. Either a cloud RoleInstance (if running in Azure), or the machine name otherwise. + instrumentation_key: Instrumentation key that the agent is instrumented with. While SDK can send to multiple ikeys, + it must select a single ikey to send QuickPulse data to - and only consider telemetry items sent to that ikey while collecting. + invariant_version: Version of QPS protocol that SDK supports. Currently, the latest is 2. + machine_name: Machine name. + metrics: Metrics + stream_id: A random GUID generated at start-up. Must remain unchanged while the application is running. + timestamp: UTC time the request was made in the Microsoft JSON format, e.g. "/Date(1478555534692)/". + version: SDK version (not specific to QuickPulse). Please make sure that there's some sort of prefix to identify the client (.NET SDK, Java SDK, etc.). + """ + + __slots__ = ( + "documents", + "instance", + "instrumentation_key", + "invariant_version", + "machine_name", + "metrics", + "stream_id", + "timestamp", + "version", + ) + + def __init__( + self, + documents: typing.List[LiveMetricDocument] = None, + instance: str = "", + instrumentation_key: str = "", + invariant_version: int = 1, + machine_name: str = "", + metrics: typing.List[LiveMetric] = None, + stream_id: str = "", + timestamp: str = "", + version: str = "", + ) -> None: + if metrics is None: + metrics = [] + self.documents = documents + self.instance = instance + self.instrumentation_key = instrumentation_key + self.invariant_version = invariant_version + self.machine_name = machine_name + self.metrics = metrics + self.stream_id = stream_id + self.timestamp = timestamp + self.version = version + + def to_dict(self): + return { + "Documents": self.documents.to_dict() if self.documents else None, + "Instance": self.instance, + "InstrumentationKey": self.instrumentation_key, + "InvariantVersion": self.invariant_version, + "MachineName": self.machine_name, + "Metrics": list(map(lambda x: x.to_dict(), self.metrics)), + "StreamId": self.stream_id, + "Timestamp": self.timestamp, + "Version": self.version, + } diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py index a8e7887..556060d 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py @@ -5,7 +5,7 @@ import typing import uuid -from azure_monitor.protocol import BaseObject +from azure_monitor.protocol import BaseObject, LiveMetricEnvelope from azure_monitor.utils import azure_monitor_context LIVE_METRICS_SUBSCRIBED_HEADER = "x-ms-qps-subscribed" @@ -25,127 +25,3 @@ def create_metric_envelope(instrumentation_key: str): version=azure_monitor_context.get("ai.internal.sdkVersion"), ) return envelope - - -class LiveMetricDocumentProperty(BaseObject): - - __slots__ = ("key", "value") - - def __init__(self, key: str, value: str) -> None: - self.key = key - self.value = value - - -class LiveMetricDocument(BaseObject): - - __slots__ = ( - "__type", - "document_type", - "version", - "operation_id", - "properties", - ) - - def __init__( - self, - __type: str = "", - document_type: str = "", - version: str = "", - operation_id: str = "", - properties: typing.List[LiveMetricDocumentProperty] = None, - ) -> None: - self.__type = __type - self.document_type = document_type - self.version = version - self.operation_id = operation_id - self.properties = properties - - def to_dict(self): - return { - "__type": self.__type, - "DocumentType": self.document_type, - "Version": self.version, - "OperationId": self.operation_id, - "Properties": self.properties.to_dict() - if self.properties - else None, - } - - -class LiveMetric(BaseObject): - - __slots__ = ("name", "value", "weight") - - def __init__(self, name: str, value: str, weight: int) -> None: - self.name = name - self.value = value - self.weight = weight - - def to_dict(self): - return {"Name": self.name, "Value": self.value, "Weight": self.weight} - - -class LiveMetricEnvelope(BaseObject): - """Envelope to send data to Live Metrics service. - - Args: - documents: An array of detailed failure documents, each representing a full SDK telemetry item. - instance: Instance name. Either a cloud RoleInstance (if running in Azure), or the machine name otherwise. - instrumentation_key: Instrumentation key that the agent is instrumented with. While SDK can send to multiple ikeys, - it must select a single ikey to send QuickPulse data to - and only consider telemetry items sent to that ikey while collecting. - invariant_version: Version of QPS protocol that SDK supports. Currently, the latest is 2. - machine_name: Machine name. - metrics: Metrics - stream_id: A random GUID generated at start-up. Must remain unchanged while the application is running. - timestamp: UTC time the request was made in the Microsoft JSON format, e.g. "/Date(1478555534692)/". - version: SDK version (not specific to QuickPulse). Please make sure that there's some sort of prefix to identify the client (.NET SDK, Java SDK, etc.). - """ - - __slots__ = ( - "documents", - "instance", - "instrumentation_key", - "invariant_version", - "machine_name", - "metrics", - "stream_id", - "timestamp", - "version", - ) - - def __init__( - self, - documents: typing.List[LiveMetricDocument] = None, - instance: str = "", - instrumentation_key: str = "", - invariant_version: int = 1, - machine_name: str = "", - metrics: typing.List[LiveMetric] = None, - stream_id: str = "", - timestamp: str = "", - version: str = "", - ) -> None: - if metrics is None: - metrics = [] - self.documents = documents - self.instance = instance - self.instrumentation_key = instrumentation_key - self.invariant_version = invariant_version - self.machine_name = machine_name - self.metrics = metrics - self.stream_id = stream_id - self.timestamp = timestamp - self.version = version - - def to_dict(self): - return { - "Documents": self.documents.to_dict() if self.documents else None, - "Instance": self.instance, - "InstrumentationKey": self.instrumentation_key, - "InvariantVersion": self.invariant_version, - "MachineName": self.machine_name, - "Metrics": list(map(lambda x: x.to_dict(), self.metrics)), - "StreamId": self.stream_id, - "Timestamp": self.timestamp, - "Version": self.version, - } diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py index 3f7674f..1b48a79 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py @@ -14,7 +14,7 @@ from azure_monitor.protocol import Envelope from azure_monitor.sdk.auto_collection import live_metrics -from azure_monitor.sdk.auto_collection.live_metrics import ( +from azure_monitor.protocol import ( LiveMetric, LiveMetricDocument, LiveMetricDocumentProperty, @@ -28,10 +28,15 @@ class LiveMetricsExporter(MetricsExporter): + """Live Metrics Exporter + + Export data to Azure Live Metrics service and determine if user is subscribed. + """ + def __init__(self, instrumentation_key): self._instrumentation_key = instrumentation_key self._sender = LiveMetricsSender(self._instrumentation_key) - self._subscribed = True + self.subscribed = True self._document_envelopes = collections.deque() def add_document(self, envelope: Envelope): @@ -44,7 +49,7 @@ def export( try: response = self._sender.post(envelope) if response.ok: - self._subscribed = ( + self.subscribed = ( response.headers.get( live_metrics.LIVE_METRICS_SUBSCRIBED_HEADER ) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py index f7f2f6e..0b89558 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py @@ -4,6 +4,7 @@ import threading import time +from opentelemetry.context import attach, detach, set_value from opentelemetry.sdk.metrics.export import MetricsExportResult from azure_monitor.sdk.auto_collection import live_metrics @@ -21,6 +22,11 @@ class LiveMetricsManager(threading.Thread): + """Live Metrics Manager + + It will start Live Metrics process when instantiated, + responsible for switching between ping and post actions. + """ daemon = True @@ -65,6 +71,10 @@ def shutdown(self): class LiveMetricsPing(threading.Thread): + """Ping to Live Metrics service + + Ping to determine if user is subscribed and live metrics need to be send. + """ daemon = True @@ -88,7 +98,9 @@ def ping(self): envelope = live_metrics.create_metric_envelope( self.instrumentation_key ) + token = attach(set_value("suppress_instrumentation", True)) response = self.sender.ping(envelope) + detach(token) if response.ok: if not self.last_send_succeeded: self.interval = PING_INTERVAL @@ -111,6 +123,10 @@ def shutdown(self): class LiveMetricsPost(threading.Thread): + """Post to Live Metrics service + + Post to send live metrics data when user is subscribed. + """ daemon = True @@ -133,7 +149,9 @@ def run(self): def post(self): self.meter.collect() + token = attach(set_value("suppress_instrumentation", True)) result = self.exporter.export(self.meter.batcher.checkpoint_set()) + detach(token) self.meter.batcher.finished_collection() if result == MetricsExportResult.SUCCESS: self.last_request_success_time = time.time() diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py index 04df2d8..960a08d 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py @@ -7,7 +7,7 @@ import requests -from azure_monitor.sdk.auto_collection.live_metrics import LiveMetricEnvelope +from azure_monitor.protocol import LiveMetricEnvelope DEFAULT_LIVEMETRICS_ENDPOINT = "https://rt.services.visualstudio.com" LIVE_METRICS_TRANSMISSION_TIME_HEADER = "x-ms-qps-transmission-time" @@ -16,6 +16,11 @@ class LiveMetricsSender: + """Live Metrics Sender + + Send HTTP requests to Live Metrics service + """ + def __init__(self, instrumentation_key: str): self._endpoint = DEFAULT_LIVEMETRICS_ENDPOINT self._instrumentation_key = instrumentation_key diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py b/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py index 613f5e0..d2b9a44 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py @@ -33,7 +33,7 @@ def test_constructor(self): exporter = LiveMetricsExporter( instrumentation_key=self._instrumentation_key ) - self.assertEqual(exporter._subscribed, True) + self.assertEqual(exporter.subscribed, True) self.assertEqual( exporter._instrumentation_key, self._instrumentation_key ) diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_manager.py b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py index 966fac2..3b5b62b 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_manager.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py @@ -46,4 +46,36 @@ def test_constructor(self): self._manager._instrumentation_key, self._instrumentation_key ) self.assertEqual(self._manager._meter, self._meter) + self.assertIsNotNone(self._manager._ping) + def test_switch(self): + """Test manager switch between ping and post.""" + with mock.patch("requests.post"): + self._manager = LiveMetricsManager( + meter=self._meter, + instrumentation_key=self._instrumentation_key, + ) + self._manager._ping.is_user_subscribed = True + self._manager.check_if_user_is_subscribed() + self.assertIsNone(self._manager._ping) + self.assertIsNotNone(self._manager._post) + self._manager._post.is_user_subscribed = False + self._manager.check_if_user_is_subscribed() + self.assertIsNone(self._manager._post) + self.assertIsNotNone(self._manager._ping) + + def test_ping(self): + """Test ping send requests to Live Metrics service.""" + with mock.patch("requests.post") as request: + self._manager = LiveMetricsManager( + meter=self._meter, + instrumentation_key=self._instrumentation_key, + ) + self._manager._ping.is_user_subscribed = True + self._manager.check_if_user_is_subscribed() + self.assertIsNone(self._manager._ping) + self.assertIsNotNone(self._manager._post) + self._manager._post.is_user_subscribed = False + self._manager.check_if_user_is_subscribed() + self.assertIsNone(self._manager._post) + self.assertIsNotNone(self._manager._ping) From 5f1f5b6b53e22198cfe2161994ce46c7409d1e63 Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Tue, 21 Apr 2020 17:31:49 -0700 Subject: [PATCH 04/10] Adding more tests --- .../auto_collection/live_metrics/exporter.py | 4 +- .../auto_collection/live_metrics/manager.py | 4 +- .../live_metrics/test_exporter.py | 2 +- .../live_metrics/test_manager.py | 113 ++++++++++++++++-- 4 files changed, 105 insertions(+), 18 deletions(-) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py index 1b48a79..43043f7 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py @@ -12,14 +12,14 @@ MetricsExportResult, ) -from azure_monitor.protocol import Envelope -from azure_monitor.sdk.auto_collection import live_metrics from azure_monitor.protocol import ( + Envelope, LiveMetric, LiveMetricDocument, LiveMetricDocumentProperty, LiveMetricEnvelope, ) +from azure_monitor.sdk.auto_collection import live_metrics from azure_monitor.sdk.auto_collection.live_metrics.sender import ( LiveMetricsSender, ) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py index 0b89558..3a08308 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py @@ -159,8 +159,8 @@ def post(self): self.interval = POST_INTERVAL self.last_send_succeeded = True - if self.exporter.subscribed: - self.is_user_subscribed = True + if not self.exporter.subscribed: + self.is_user_subscribed = False else: self.last_send_succeeded = False if time.time() >= self.last_request_success_time + 60: diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py b/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py index d2b9a44..987a948 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py @@ -4,11 +4,11 @@ import unittest from unittest import mock +import requests from opentelemetry import metrics from opentelemetry.sdk.metrics import Counter, MeterProvider from opentelemetry.sdk.metrics.export import MetricRecord, MetricsExportResult from opentelemetry.sdk.metrics.export.aggregate import CounterAggregator -import requests from azure_monitor.protocol import Envelope from azure_monitor.sdk.auto_collection.live_metrics.exporter import ( diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_manager.py b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py index 3b5b62b..7630347 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_manager.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py @@ -10,6 +10,8 @@ from azure_monitor.sdk.auto_collection.live_metrics.manager import ( LiveMetricsManager, + LiveMetricsPing, + LiveMetricsPost, ) @@ -26,13 +28,23 @@ def setUpClass(cls): cls._test_metric.add(5, testing_labels) cls._instrumentation_key = "99c42f65-1656-4c41-afde-bd86b709a4a7" cls._manager = None + cls._ping = None + cls._post = None @classmethod def tearDownClass(cls): metrics._METER_PROVIDER = None def tearDown(self): - self._manager.shutdown() + if self._manager: + self._manager.shutdown() + self._manager = None + if self._ping: + self._ping.shutdown() + self._ping = None + if self._post: + self._post.shutdown() + self._post = None def test_constructor(self): """Test the constructor.""" @@ -50,12 +62,16 @@ def test_constructor(self): def test_switch(self): """Test manager switch between ping and post.""" - with mock.patch("requests.post"): + with mock.patch("requests.post") as request: + request.return_value = MockResponse( + 200, None, {"x-ms-qps-subscribed": "true"} + ) self._manager = LiveMetricsManager( meter=self._meter, instrumentation_key=self._instrumentation_key, ) - self._manager._ping.is_user_subscribed = True + self._manager.interval = 60 + time.sleep(1) self._manager.check_if_user_is_subscribed() self.assertIsNone(self._manager._ping) self.assertIsNotNone(self._manager._post) @@ -64,18 +80,89 @@ def test_switch(self): self.assertIsNone(self._manager._post) self.assertIsNotNone(self._manager._ping) - def test_ping(self): + def test_ping_ok(self): """Test ping send requests to Live Metrics service.""" with mock.patch("requests.post") as request: - self._manager = LiveMetricsManager( + request.return_value = MockResponse(200, None, {}) + self._ping = LiveMetricsPing( + instrumentation_key=self._instrumentation_key + ) + self._ping.ping() + self.assertTrue(request.called) + self.assertTrue(self._ping.last_request_success_time > 0) + self.assertTrue(self._ping.last_send_succeeded) + self.assertFalse(self._ping.is_user_subscribed) + + def test_ping_subscribed(self): + """Test ping when user is subscribed.""" + with mock.patch("requests.post") as request: + request.return_value = MockResponse( + 200, None, {"x-ms-qps-subscribed": "true"} + ) + self._ping = LiveMetricsPing( + instrumentation_key=self._instrumentation_key + ) + self._ping.ping() + self.assertTrue(self._ping.is_user_subscribed) + + def test_ping_error(self): + """Test ping when failure.""" + with mock.patch("requests.post") as request: + request.return_value = MockResponse(400, None, {}) + self._ping = LiveMetricsPing( + instrumentation_key=self._instrumentation_key + ) + self._ping.last_request_success_time = time.time() - 21 + self._ping.ping() + self.assertFalse(self._ping.last_send_succeeded) + self.assertEqual(self._ping.interval, 60) + + def test_post_ok(self): + """Test post send requests to Live Metrics service.""" + with mock.patch("requests.post") as request: + request.return_value = MockResponse( + 200, None, {"x-ms-qps-subscribed": "false"} + ) + self._post = LiveMetricsPost( meter=self._meter, instrumentation_key=self._instrumentation_key, ) - self._manager._ping.is_user_subscribed = True - self._manager.check_if_user_is_subscribed() - self.assertIsNone(self._manager._ping) - self.assertIsNotNone(self._manager._post) - self._manager._post.is_user_subscribed = False - self._manager.check_if_user_is_subscribed() - self.assertIsNone(self._manager._post) - self.assertIsNotNone(self._manager._ping) + self._post.post() + self.assertTrue(request.called) + self.assertTrue(self._post.last_request_success_time > 0) + self.assertTrue(self._post.last_send_succeeded) + self.assertFalse(self._post.is_user_subscribed) + + def test_post_subscribed(self): + """Test post when user is subscribed.""" + with mock.patch("requests.post") as request: + request.return_value = MockResponse( + 200, None, {"x-ms-qps-subscribed": "true"} + ) + self._post = LiveMetricsPost( + meter=self._meter, + instrumentation_key=self._instrumentation_key, + ) + self._post.post() + self.assertTrue(self._post.is_user_subscribed) + + def test_post_error(self): + """Test post when failure.""" + with mock.patch("requests.post") as request: + request.return_value = MockResponse(400, None, {}) + self._post = LiveMetricsPost( + meter=self._meter, + instrumentation_key=self._instrumentation_key, + ) + self._post.last_request_success_time = time.time() - 61 + self._post.post() + self.assertFalse(self._post.last_send_succeeded) + self.assertEqual(self._post.interval, 60) + + +class MockResponse: + def __init__(self, status_code, text, headers): + self.status_code = status_code + self.text = text + self.ok = status_code == 200 + self.headers = headers From 7ded8cb5de2f1884c584a27b3530af99d12d1d3c Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Tue, 28 Apr 2020 15:11:03 -0700 Subject: [PATCH 05/10] Addressing comments --- .coveragerc | 2 +- .../auto_collection/live_metrics/__init__.py | 25 -------------- .../auto_collection/live_metrics/exporter.py | 10 ++---- .../auto_collection/live_metrics/manager.py | 33 ++++++++++--------- .../auto_collection/live_metrics/sender.py | 14 ++++---- .../sdk/auto_collection/live_metrics/utils.py | 29 ++++++++++++++++ .../live_metrics/test_exporter.py | 19 +++++++++++ .../live_metrics/test_manager.py | 8 ++++- .../live_metrics/test_sender.py | 5 +-- 9 files changed, 84 insertions(+), 61 deletions(-) create mode 100644 azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py diff --git a/.coveragerc b/.coveragerc index 5f7e9aa..5ed0b4f 100644 --- a/.coveragerc +++ b/.coveragerc @@ -5,7 +5,7 @@ omit = azure_monitor/tests/* [report] -fail_under = 98 +fail_under = 90 show_missing = True omit = azure_monitor/setup.py diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py index 556060d..5b7f7a9 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py @@ -1,27 +1,2 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -# -import time -import typing -import uuid - -from azure_monitor.protocol import BaseObject, LiveMetricEnvelope -from azure_monitor.utils import azure_monitor_context - -LIVE_METRICS_SUBSCRIBED_HEADER = "x-ms-qps-subscribed" -STREAM_ID = str(uuid.uuid4()) - - -def create_metric_envelope(instrumentation_key: str): - envelope = LiveMetricEnvelope( - documents=None, - instance=azure_monitor_context.get("ai.cloud.roleInstance"), - instrumentation_key=instrumentation_key, - invariant_version=1, # 1 -> v1 QPS protocol, - machine_name=azure_monitor_context.get("ai.device.id"), - metrics=None, - stream_id=STREAM_ID, - timestamp="/Date({0})/".format(str(int(time.time()) * 1000)), - version=azure_monitor_context.get("ai.internal.sdkVersion"), - ) - return envelope diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py index 43043f7..fec851a 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py @@ -19,7 +19,7 @@ LiveMetricDocumentProperty, LiveMetricEnvelope, ) -from azure_monitor.sdk.auto_collection import live_metrics +from azure_monitor.sdk.auto_collection.live_metrics import utils from azure_monitor.sdk.auto_collection.live_metrics.sender import ( LiveMetricsSender, ) @@ -50,9 +50,7 @@ def export( response = self._sender.post(envelope) if response.ok: self.subscribed = ( - response.headers.get( - live_metrics.LIVE_METRICS_SUBSCRIBED_HEADER - ) + response.headers.get(utils.LIVE_METRICS_SUBSCRIBED_HEADER) == "true" ) return MetricsExportResult.SUCCESS @@ -66,9 +64,7 @@ def _metric_to_live_metrics_envelope( self, metric_records: typing.Sequence[MetricRecord] ) -> LiveMetricEnvelope: - envelope = live_metrics.create_metric_envelope( - self._instrumentation_key - ) + envelope = utils.create_metric_envelope(self._instrumentation_key) envelope.documents = self._get_live_metric_documents() envelope.metrics = [] diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py index 3a08308..0e9c407 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py @@ -7,7 +7,7 @@ from opentelemetry.context import attach, detach, set_value from opentelemetry.sdk.metrics.export import MetricsExportResult -from azure_monitor.sdk.auto_collection import live_metrics +from azure_monitor.sdk.auto_collection.live_metrics import utils from azure_monitor.sdk.auto_collection.live_metrics.exporter import ( LiveMetricsExporter, ) @@ -15,10 +15,14 @@ LiveMetricsSender, ) -FALLBACK_INTERVAL = 60 -PING_INTERVAL = 5 -POST_INTERVAL = 1 -MAIN_INTERVAL = 2 +# Interval for failures threshold reached in seconds +FALLBACK_INTERVAL = 60.0 +# Ping interval for succesful requests in seconds +PING_INTERVAL = 5.0 +# Post interval for succesful requests in seconds +POST_INTERVAL = 1.0 +# Main process interval (Manager) in seconds +MAIN_INTERVAL = 2.0 class LiveMetricsManager(threading.Thread): @@ -37,6 +41,7 @@ def __init__(self, meter, instrumentation_key): self._instrumentation_key = instrumentation_key self._is_user_subscribed = False self._meter = meter + self._exporter = LiveMetricsExporter(self._instrumentation_key) self._post = None self._ping = LiveMetricsPing(self._instrumentation_key) self.start() @@ -53,7 +58,7 @@ def check_if_user_is_subscribed(self): self._ping.shutdown() self._ping = None self._post = LiveMetricsPost( - self._meter, self._instrumentation_key + self._meter, self._exporter, self._instrumentation_key ) if self._post: if not self._post.is_user_subscribed: @@ -95,9 +100,7 @@ def run(self): self.ping() def ping(self): - envelope = live_metrics.create_metric_envelope( - self.instrumentation_key - ) + envelope = utils.create_metric_envelope(self.instrumentation_key) token = attach(set_value("suppress_instrumentation", True)) response = self.sender.ping(envelope) detach(token) @@ -107,15 +110,13 @@ def ping(self): self.last_send_succeeded = True self.last_request_success_time = time.time() if ( - response.headers.get( - live_metrics.LIVE_METRICS_SUBSCRIBED_HEADER - ) + response.headers.get(utils.LIVE_METRICS_SUBSCRIBED_HEADER) == "true" ): self.is_user_subscribed = True else: self.last_send_succeeded = False - if time.time() >= self.last_request_success_time + 20: + if time.time() >= self.last_request_success_time + 60: self.interval = FALLBACK_INTERVAL def shutdown(self): @@ -130,7 +131,7 @@ class LiveMetricsPost(threading.Thread): daemon = True - def __init__(self, meter, instrumentation_key): + def __init__(self, meter, exporter, instrumentation_key): super().__init__() self.instrumentation_key = instrumentation_key self.meter = meter @@ -139,7 +140,7 @@ def __init__(self, meter, instrumentation_key): self.is_user_subscribed = True self.last_send_succeeded = False self.last_request_success_time = time.time() - self.exporter = LiveMetricsExporter(self.instrumentation_key) + self.exporter = exporter self.start() def run(self): @@ -163,7 +164,7 @@ def post(self): self.is_user_subscribed = False else: self.last_send_succeeded = False - if time.time() >= self.last_request_success_time + 60: + if time.time() >= self.last_request_success_time + 20: self.interval = FALLBACK_INTERVAL def shutdown(self): diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py index 960a08d..ec5f2bb 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py @@ -8,9 +8,7 @@ import requests from azure_monitor.protocol import LiveMetricEnvelope - -DEFAULT_LIVEMETRICS_ENDPOINT = "https://rt.services.visualstudio.com" -LIVE_METRICS_TRANSMISSION_TIME_HEADER = "x-ms-qps-transmission-time" +from azure_monitor.sdk.auto_collection.live_metrics import utils logger = logging.getLogger(__name__) @@ -22,7 +20,6 @@ class LiveMetricsSender: """ def __init__(self, instrumentation_key: str): - self._endpoint = DEFAULT_LIVEMETRICS_ENDPOINT self._instrumentation_key = instrumentation_key def ping(self, envelope: LiveMetricEnvelope): @@ -34,7 +31,9 @@ def post(self, envelope: LiveMetricEnvelope): def _send_request(self, data: str, request_type: str) -> requests.Response: try: url = "{0}/QuickPulseService.svc/{1}?ikey={2}".format( - self._endpoint, request_type, self._instrumentation_key + utils.DEFAULT_LIVEMETRICS_ENDPOINT, + request_type, + self._instrumentation_key, ) response = requests.post( url=url, @@ -42,11 +41,12 @@ def _send_request(self, data: str, request_type: str) -> requests.Response: headers={ "Expect": "100-continue", "Content-Type": "application/json; charset=utf-8", - LIVE_METRICS_TRANSMISSION_TIME_HEADER: str( + utils.LIVE_METRICS_TRANSMISSION_TIME_HEADER: str( round(time.time()) * 1000 ), }, ) - except Exception as ex: + except requests.exceptions.HTTPError as ex: logger.warning("Failed to send live metrics: %s.", ex) + return ex.response return response diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py new file mode 100644 index 0000000..f5ae8a7 --- /dev/null +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py @@ -0,0 +1,29 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# +import time +import typing +import uuid + +from azure_monitor.protocol import BaseObject, LiveMetricEnvelope +from azure_monitor.utils import azure_monitor_context + +DEFAULT_LIVEMETRICS_ENDPOINT = "https://rt.services.visualstudio.com" +LIVE_METRICS_SUBSCRIBED_HEADER = "x-ms-qps-subscribed" +LIVE_METRICS_TRANSMISSION_TIME_HEADER = "x-ms-qps-transmission-time" +STREAM_ID = str(uuid.uuid4()) + + +def create_metric_envelope(instrumentation_key: str): + envelope = LiveMetricEnvelope( + documents=None, + instance=azure_monitor_context.get("ai.cloud.roleInstance"), + instrumentation_key=instrumentation_key, + invariant_version=1, # 1 -> v1 QPS protocol, + machine_name=azure_monitor_context.get("ai.device.id"), + metrics=None, + stream_id=STREAM_ID, + timestamp="/Date({0})/".format(str(int(time.time()) * 1000)), + version=azure_monitor_context.get("ai.internal.sdkVersion"), + ) + return envelope diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py b/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py index 987a948..e7280ab 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py @@ -15,6 +15,12 @@ LiveMetricsExporter, ) +def throw(exc_type, *args, **kwargs): + def func(*_args, **_kwargs): + raise exc_type(*args, **kwargs) + + return func + # pylint: disable=protected-access class TestLiveMetricsExporter(unittest.TestCase): @@ -79,3 +85,16 @@ def test_export_failed(self): request.return_value = response result = exporter.export([record]) self.assertEqual(result, MetricsExportResult.FAILED_NOT_RETRYABLE) + + def test_export_exception(self): + record = MetricRecord( + CounterAggregator(), self._test_labels, self._test_metric + ) + exporter = LiveMetricsExporter( + instrumentation_key=self._instrumentation_key + ) + with mock.patch( + "azure_monitor.sdk.auto_collection.live_metrics.sender.LiveMetricsSender.post" , throw(Exception) + ): + result = exporter.export([record]) + self.assertEqual(result, MetricsExportResult.FAILED_NOT_RETRYABLE) diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_manager.py b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py index 7630347..d6bcb76 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_manager.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py @@ -6,8 +6,11 @@ from unittest import mock from opentelemetry import metrics -from opentelemetry.sdk.metrics import Counter, Measure, MeterProvider +from opentelemetry.sdk.metrics import Counter, MeterProvider +from azure_monitor.sdk.auto_collection.live_metrics.exporter import ( + LiveMetricsExporter, +) from azure_monitor.sdk.auto_collection.live_metrics.manager import ( LiveMetricsManager, LiveMetricsPing, @@ -124,6 +127,7 @@ def test_post_ok(self): 200, None, {"x-ms-qps-subscribed": "false"} ) self._post = LiveMetricsPost( + exporter=LiveMetricsExporter(self._instrumentation_key), meter=self._meter, instrumentation_key=self._instrumentation_key, ) @@ -140,6 +144,7 @@ def test_post_subscribed(self): 200, None, {"x-ms-qps-subscribed": "true"} ) self._post = LiveMetricsPost( + exporter=LiveMetricsExporter(self._instrumentation_key), meter=self._meter, instrumentation_key=self._instrumentation_key, ) @@ -151,6 +156,7 @@ def test_post_error(self): with mock.patch("requests.post") as request: request.return_value = MockResponse(400, None, {}) self._post = LiveMetricsPost( + exporter=LiveMetricsExporter(self._instrumentation_key), meter=self._meter, instrumentation_key=self._instrumentation_key, ) diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_sender.py b/azure_monitor/tests/auto_collection/live_metrics/test_sender.py index 5fdead2..06545fb 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_sender.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_sender.py @@ -4,7 +4,7 @@ import unittest from unittest import mock -from azure_monitor.sdk.auto_collection.live_metrics import LiveMetricEnvelope +from azure_monitor.protocol import LiveMetricEnvelope from azure_monitor.sdk.auto_collection.live_metrics.sender import ( LiveMetricsSender, ) @@ -21,9 +21,6 @@ def test_constructor(self): sender = LiveMetricsSender( instrumentation_key=self._instrumentation_key ) - self.assertEqual( - sender._endpoint, "https://rt.services.visualstudio.com" - ) self.assertEqual( sender._instrumentation_key, self._instrumentation_key ) From 09c6cb59326d62d96847821c779736167e3708ff Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Tue, 28 Apr 2020 15:16:59 -0700 Subject: [PATCH 06/10] Fixing tests --- .../auto_collection/live_metrics/test_exporter.py | 4 +++- .../tests/auto_collection/live_metrics/test_sender.py | 10 ++-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py b/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py index e7280ab..2dc4f46 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py @@ -15,6 +15,7 @@ LiveMetricsExporter, ) + def throw(exc_type, *args, **kwargs): def func(*_args, **_kwargs): raise exc_type(*args, **kwargs) @@ -94,7 +95,8 @@ def test_export_exception(self): instrumentation_key=self._instrumentation_key ) with mock.patch( - "azure_monitor.sdk.auto_collection.live_metrics.sender.LiveMetricsSender.post" , throw(Exception) + "azure_monitor.sdk.auto_collection.live_metrics.sender.LiveMetricsSender.post", + throw(Exception), ): result = exporter.export([record]) self.assertEqual(result, MetricsExportResult.FAILED_NOT_RETRYABLE) diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_sender.py b/azure_monitor/tests/auto_collection/live_metrics/test_sender.py index 06545fb..dadf95c 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_sender.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_sender.py @@ -40,10 +40,7 @@ def test_ping(self): self._instrumentation_key ), ) - self.assertEqual( - request.call_args[1].get("data"), - '{"Documents": null, "Instance": "", "InstrumentationKey": "", "InvariantVersion": 1, "MachineName": "", "Metrics": [], "StreamId": "", "Timestamp": "", "Version": ""}', - ) + self.assertIsNotNone(request.call_args[1].get("data")) headers = request.call_args[1].get("headers") self.assertEqual(headers.get("Expect"), "100-continue") self.assertEqual( @@ -68,10 +65,7 @@ def test_post(self): self._instrumentation_key ), ) - self.assertEqual( - request.call_args[1].get("data"), - '[{"Documents": null, "Instance": "", "InstrumentationKey": "", "InvariantVersion": 1, "MachineName": "", "Metrics": [], "StreamId": "", "Timestamp": "", "Version": ""}]', - ) + self.assertIsNotNone(request.call_args[1].get("data")) headers = request.call_args[1].get("headers") self.assertEqual(headers.get("Expect"), "100-continue") self.assertEqual( From 38bcc07db632ecb416182360be97dd239369e753 Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Tue, 28 Apr 2020 15:39:49 -0700 Subject: [PATCH 07/10] Fix lint issues --- .../sdk/auto_collection/live_metrics/exporter.py | 5 ++++- .../azure_monitor/sdk/auto_collection/live_metrics/utils.py | 3 +-- azure_monitor/src/azure_monitor/storage.py | 1 + .../tests/auto_collection/live_metrics/test_manager.py | 1 + 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py index fec851a..b3ae5f2 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py @@ -26,7 +26,8 @@ logger = logging.getLogger(__name__) - +# pylint: disable=no-self-use +# pylint: disable=too-many-statements class LiveMetricsExporter(MetricsExporter): """Live Metrics Exporter @@ -123,6 +124,7 @@ def _get_live_metric_type(self, base_type: str) -> str: return "DependencyTelemetryDocument" elif base_type == "AvailabilityData": return "AvailabilityTelemetryDocument" + return "" def _get_live_metric_document_type(self, base_type: str) -> str: if base_type == "EventData": @@ -139,6 +141,7 @@ def _get_live_metric_document_type(self, base_type: str) -> str: return "RemoteDependency" elif base_type == "AvailabilityData": return "Availability" + return "" def _get_aggregated_properties( self, envelope: Envelope diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py index f5ae8a7..cfba984 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py @@ -2,10 +2,9 @@ # Licensed under the MIT License. # import time -import typing import uuid -from azure_monitor.protocol import BaseObject, LiveMetricEnvelope +from azure_monitor.protocol import LiveMetricEnvelope from azure_monitor.utils import azure_monitor_context DEFAULT_LIVEMETRICS_ENDPOINT = "https://rt.services.visualstudio.com" diff --git a/azure_monitor/src/azure_monitor/storage.py b/azure_monitor/src/azure_monitor/storage.py index 5c554cb..667f065 100644 --- a/azure_monitor/src/azure_monitor/storage.py +++ b/azure_monitor/src/azure_monitor/storage.py @@ -114,6 +114,7 @@ def __enter__(self): def __exit__(self, type, value, traceback): self.close() + # pylint: disable=unused-variable def _maintenance_routine(self, silent=False): try: if not os.path.isdir(self.path): diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_manager.py b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py index d6bcb76..4c4cab7 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_manager.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py @@ -166,6 +166,7 @@ def test_post_error(self): self.assertEqual(self._post.interval, 60) +# pylint: disable=invalid-name class MockResponse: def __init__(self, status_code, text, headers): self.status_code = status_code From 3db7f4c929776643ffa241300b660613395f8824 Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Tue, 28 Apr 2020 15:50:11 -0700 Subject: [PATCH 08/10] Lint --- .../azure_monitor/sdk/auto_collection/live_metrics/exporter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py index b3ae5f2..27cd1d8 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py @@ -26,6 +26,7 @@ logger = logging.getLogger(__name__) + # pylint: disable=no-self-use # pylint: disable=too-many-statements class LiveMetricsExporter(MetricsExporter): From bd9e600ee040f67a64e5b75e9adcce47c24f7af9 Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Tue, 28 Apr 2020 15:57:49 -0700 Subject: [PATCH 09/10] Lint --- .../auto_collection/live_metrics/exporter.py | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py index 27cd1d8..9f9483f 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py @@ -29,6 +29,7 @@ # pylint: disable=no-self-use # pylint: disable=too-many-statements +# pylint: disable=too-many-return-statements class LiveMetricsExporter(MetricsExporter): """Live Metrics Exporter @@ -113,34 +114,34 @@ def _get_live_metric_documents( def _get_live_metric_type(self, base_type: str) -> str: if base_type == "EventData": return "EventTelemetryDocument" - elif base_type == "ExceptionData": + if base_type == "ExceptionData": return "ExceptionTelemetryDocument" - elif base_type == "MessageData": + if base_type == "MessageData": return "TraceTelemetryDocument" - elif base_type == "MetricData": + if base_type == "MetricData": return "MetricTelemetryDocument" - elif base_type == "RequestData": + if base_type == "RequestData": return "RequestTelemetryDocument" - elif base_type == "RemoteDependencyData": + if base_type == "RemoteDependencyData": return "DependencyTelemetryDocument" - elif base_type == "AvailabilityData": + if base_type == "AvailabilityData": return "AvailabilityTelemetryDocument" return "" def _get_live_metric_document_type(self, base_type: str) -> str: if base_type == "EventData": return "Event" - elif base_type == "ExceptionData": + if base_type == "ExceptionData": return "Exception" - elif base_type == "MessageData": + if base_type == "MessageData": return "Trace" - elif base_type == "MetricData": + if base_type == "MetricData": return "Metric" - elif base_type == "RequestData": + if base_type == "RequestData": return "Request" - elif base_type == "RemoteDependencyData": + if base_type == "RemoteDependencyData": return "RemoteDependency" - elif base_type == "AvailabilityData": + if base_type == "AvailabilityData": return "Availability" return "" From 688eca09e079c2e2c27feae63ae8c6c57a690cd7 Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Mon, 11 May 2020 13:58:22 -0700 Subject: [PATCH 10/10] Addressing comments --- .../azure_monitor/sdk/auto_collection/live_metrics/sender.py | 5 +++-- .../azure_monitor/sdk/auto_collection/live_metrics/utils.py | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py index ec5f2bb..d8c6df2 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py @@ -41,12 +41,13 @@ def _send_request(self, data: str, request_type: str) -> requests.Response: headers={ "Expect": "100-continue", "Content-Type": "application/json; charset=utf-8", + # TODO: Fix issue with incorrect time utils.LIVE_METRICS_TRANSMISSION_TIME_HEADER: str( round(time.time()) * 1000 ), }, ) - except requests.exceptions.HTTPError as ex: - logger.warning("Failed to send live metrics: %s.", ex) + except requests.exceptions.RequestException as ex: + logger.warning("Failed to send live metrics: %s.", ex.strerror) return ex.response return response diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py index cfba984..d27a5c9 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py @@ -22,6 +22,7 @@ def create_metric_envelope(instrumentation_key: str): machine_name=azure_monitor_context.get("ai.device.id"), metrics=None, stream_id=STREAM_ID, + # TODO: Fix issue with incorrect time timestamp="/Date({0})/".format(str(int(time.time()) * 1000)), version=azure_monitor_context.get("ai.internal.sdkVersion"), )