diff --git a/azure_monitor/examples/metrics/auto_collector.py b/azure_monitor/examples/metrics/auto_collector.py index 75b1230..d473c7e 100644 --- a/azure_monitor/examples/metrics/auto_collector.py +++ b/azure_monitor/examples/metrics/auto_collector.py @@ -1,11 +1,21 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from opentelemetry import metrics +from opentelemetry import metrics, trace from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export.controller import PushController +from opentelemetry.sdk.trace import TracerProvider from azure_monitor import AzureMonitorMetricsExporter -from azure_monitor.sdk.auto_collection import AutoCollection +from azure_monitor.sdk.auto_collection import ( + AutoCollection, + AzureMetricsSpanProcessor, +) + +# Add Span Processor to get metrics about traces +span_processor = AzureMetricsSpanProcessor() +tracer_provider = TracerProvider() +tracer_provider.add_span_processor(span_processor) +trace.set_tracer_provider(tracer_provider) metrics.set_meter_provider(MeterProvider()) meter = metrics.get_meter(__name__) @@ -17,7 +27,9 @@ testing_label_set = {"environment": "testing"} # Automatically collect standard metrics -auto_collection = AutoCollection(meter=meter, labels=testing_label_set) +auto_collection = AutoCollection( + meter=meter, labels=testing_label_set, span_processor=span_processor +) # To configure a separate export interval specific for standard metrics # meter_standard = metrics.get_meter(__name__ + "_standard") diff --git a/azure_monitor/src/azure_monitor/export/__init__.py b/azure_monitor/src/azure_monitor/export/__init__.py index 814d15e..8460a5d 100644 --- a/azure_monitor/src/azure_monitor/export/__init__.py +++ b/azure_monitor/src/azure_monitor/export/__init__.py @@ -4,11 +4,16 @@ import logging import typing from enum import Enum +from urllib.parse import urlparse import requests from opentelemetry.sdk.metrics.export import MetricsExportResult from opentelemetry.sdk.trace.export import SpanExportResult +from opentelemetry.sdk.util import ns_to_iso_str +from opentelemetry.trace import Span, SpanKind +from opentelemetry.trace.status import StatusCanonicalCode +from azure_monitor import protocol, utils from azure_monitor.options import ExporterOptions from azure_monitor.protocol import Envelope from azure_monitor.storage import LocalFileStorage @@ -204,3 +209,105 @@ def get_metrics_export_result(result: ExportResult) -> MetricsExportResult: ): return MetricsExportResult.FAILURE return None + + +# pylint: disable=too-many-statements +# pylint: disable=too-many-branches +def convert_span_to_envelope(span: Span) -> protocol.Envelope: + if not span: + return None + envelope = protocol.Envelope( + ikey="", + tags=dict(utils.azure_monitor_context), + time=ns_to_iso_str(span.start_time), + ) + envelope.tags["ai.operation.id"] = "{:032x}".format(span.context.trace_id) + parent = span.parent + if isinstance(parent, Span): + parent = parent.context + if parent: + envelope.tags["ai.operation.parentId"] = "{:016x}".format( + parent.span_id + ) + if span.kind in (SpanKind.CONSUMER, SpanKind.SERVER): + envelope.name = "Microsoft.ApplicationInsights.Request" + data = protocol.Request( + id="{:016x}".format(span.context.span_id), + duration=utils.ns_to_duration(span.end_time - span.start_time), + response_code=str(span.status.canonical_code.value), + success=span.status.canonical_code + == StatusCanonicalCode.OK, # Modify based off attributes or Status + properties={}, + ) + envelope.data = protocol.Data(base_data=data, base_type="RequestData") + if "http.method" in span.attributes: + data.name = span.attributes["http.method"] + if "http.route" in span.attributes: + data.name = data.name + " " + span.attributes["http.route"] + envelope.tags["ai.operation.name"] = data.name + data.properties["request.name"] = data.name + elif "http.path" in span.attributes: + data.properties["request.name"] = ( + data.name + " " + span.attributes["http.path"] + ) + if "http.url" in span.attributes: + data.url = span.attributes["http.url"] + data.properties["request.url"] = span.attributes["http.url"] + if "http.status_code" in span.attributes: + status_code = span.attributes["http.status_code"] + data.response_code = str(status_code) + data.success = 200 <= status_code < 400 + else: + envelope.name = "Microsoft.ApplicationInsights.RemoteDependency" + data = protocol.RemoteDependency( + name=span.name, + id="{:016x}".format(span.context.span_id), + result_code=str(span.status.canonical_code.value), + duration=utils.ns_to_duration(span.end_time - span.start_time), + success=span.status.canonical_code + == StatusCanonicalCode.OK, # Modify based off attributes or Status + properties={}, + ) + envelope.data = protocol.Data( + base_data=data, base_type="RemoteDependencyData" + ) + if span.kind in (SpanKind.CLIENT, SpanKind.PRODUCER): + if ( + "component" in span.attributes + and span.attributes["component"] == "http" + ): + data.type = "HTTP" + if "http.url" in span.attributes: + url = span.attributes["http.url"] + # data is the url + data.data = url + parse_url = urlparse(url) + # TODO: error handling, probably put scheme as well + # target matches authority (host:port) + data.target = parse_url.netloc + if "http.method" in span.attributes: + # name is METHOD/path + data.name = ( + span.attributes["http.method"] + "/" + parse_url.path + ) + if "http.status_code" in span.attributes: + status_code = span.attributes["http.status_code"] + data.result_code = str(status_code) + data.success = 200 <= status_code < 400 + else: # SpanKind.INTERNAL + data.type = "InProc" + data.success = True + for key in span.attributes: + # This removes redundant data from ApplicationInsights + if key.startswith("http."): + continue + data.properties[key] = span.attributes[key] + if span.links: + links = [] + for link in span.links: + operation_id = "{:032x}".format(link.context.trace_id) + span_id = "{:016x}".format(link.context.span_id) + links.append({"operation_Id": operation_id, "id": span_id}) + data.properties["_MS.links"] = json.dumps(links) + # TODO: tracestate, tags + return envelope diff --git a/azure_monitor/src/azure_monitor/export/trace/__init__.py b/azure_monitor/src/azure_monitor/export/trace/__init__.py index 805d271..1a0e32a 100644 --- a/azure_monitor/src/azure_monitor/export/trace/__init__.py +++ b/azure_monitor/src/azure_monitor/export/trace/__init__.py @@ -14,6 +14,7 @@ from azure_monitor.export import ( BaseExporter, ExportResult, + convert_span_to_envelope, get_trace_export_result, ) @@ -52,104 +53,6 @@ def export(self, spans: Sequence[Span]) -> SpanExportResult: def _span_to_envelope(self, span: Span) -> protocol.Envelope: if not span: return None - envelope = protocol.Envelope( - ikey=self.options.instrumentation_key, - tags=dict(utils.azure_monitor_context), - time=ns_to_iso_str(span.start_time), - ) - envelope.tags["ai.operation.id"] = "{:032x}".format( - span.context.trace_id - ) - parent = span.parent - if isinstance(parent, Span): - parent = parent.context - if parent: - envelope.tags["ai.operation.parentId"] = "{:016x}".format( - parent.span_id - ) - if span.kind in (SpanKind.CONSUMER, SpanKind.SERVER): - envelope.name = "Microsoft.ApplicationInsights.Request" - data = protocol.Request( - id="{:016x}".format(span.context.span_id), - duration=utils.ns_to_duration(span.end_time - span.start_time), - response_code=str(span.status.canonical_code.value), - success=span.status.canonical_code - == StatusCanonicalCode.OK, # Modify based off attributes or Status - properties={}, - ) - envelope.data = protocol.Data( - base_data=data, base_type="RequestData" - ) - if "http.method" in span.attributes: - data.name = span.attributes["http.method"] - if "http.route" in span.attributes: - data.name = data.name + " " + span.attributes["http.route"] - envelope.tags["ai.operation.name"] = data.name - data.properties["request.name"] = data.name - elif "http.path" in span.attributes: - data.properties["request.name"] = ( - data.name + " " + span.attributes["http.path"] - ) - if "http.url" in span.attributes: - data.url = span.attributes["http.url"] - data.properties["request.url"] = span.attributes["http.url"] - if "http.status_code" in span.attributes: - status_code = span.attributes["http.status_code"] - data.response_code = str(status_code) - data.success = 200 <= status_code < 400 - else: - envelope.name = "Microsoft.ApplicationInsights.RemoteDependency" - data = protocol.RemoteDependency( - name=span.name, - id="{:016x}".format(span.context.span_id), - result_code=str(span.status.canonical_code.value), - duration=utils.ns_to_duration(span.end_time - span.start_time), - success=span.status.canonical_code - == StatusCanonicalCode.OK, # Modify based off attributes or Status - properties={}, - ) - envelope.data = protocol.Data( - base_data=data, base_type="RemoteDependencyData" - ) - if span.kind in (SpanKind.CLIENT, SpanKind.PRODUCER): - if ( - "component" in span.attributes - and span.attributes["component"] == "http" - ): - data.type = "HTTP" - if "http.url" in span.attributes: - url = span.attributes["http.url"] - # data is the url - data.data = url - parse_url = urlparse(url) - # TODO: error handling, probably put scheme as well - # target matches authority (host:port) - data.target = parse_url.netloc - if "http.method" in span.attributes: - # name is METHOD/path - data.name = ( - span.attributes["http.method"] - + "/" - + parse_url.path - ) - if "http.status_code" in span.attributes: - status_code = span.attributes["http.status_code"] - data.result_code = str(status_code) - data.success = 200 <= status_code < 400 - else: # SpanKind.INTERNAL - data.type = "InProc" - data.success = True - for key in span.attributes: - # This removes redundant data from ApplicationInsights - if key.startswith("http."): - continue - data.properties[key] = span.attributes[key] - if span.links: - links = [] - for link in span.links: - operation_id = "{:032x}".format(link.context.trace_id) - span_id = "{:016x}".format(link.context.span_id) - links.append({"operation_Id": operation_id, "id": span_id}) - data.properties["_MS.links"] = json.dumps(links) - # TODO: tracestate, tags + envelope = convert_span_to_envelope(span) + envelope.ikey = self.options.instrumentation_key return envelope diff --git a/azure_monitor/src/azure_monitor/protocol.py b/azure_monitor/src/azure_monitor/protocol.py index e10c8d4..d482b77 100644 --- a/azure_monitor/src/azure_monitor/protocol.py +++ b/azure_monitor/src/azure_monitor/protocol.py @@ -563,19 +563,10 @@ def to_dict(self): } -class LiveMetricDocumentProperty(BaseObject): - - __slots__ = ("key", "value") - - def __init__(self, key: str, value: str) -> None: - self.key = key - self.value = value - - class LiveMetricDocument(BaseObject): __slots__ = ( - "__type", + "quickpulse_type", "document_type", "version", "operation_id", @@ -584,13 +575,13 @@ class LiveMetricDocument(BaseObject): def __init__( self, - __type: str = "", + quickpulse_type: str = "", document_type: str = "", version: str = "", operation_id: str = "", - properties: typing.List[LiveMetricDocumentProperty] = None, + properties: typing.Dict[str, any] = None, ) -> None: - self.__type = __type + self.quickpulse_type = quickpulse_type self.document_type = document_type self.version = version self.operation_id = operation_id @@ -598,13 +589,11 @@ def __init__( def to_dict(self): return { - "__type": self.__type, + "__type": self.quickpulse_type, "DocumentType": self.document_type, "Version": self.version, "OperationId": self.operation_id, - "Properties": self.properties.to_dict() - if self.properties - else None, + "Properties": self.properties, } @@ -675,7 +664,9 @@ def __init__( def to_dict(self): return { - "Documents": self.documents.to_dict() if self.documents else None, + "Documents": list(map(lambda x: x.to_dict(), self.documents)) + if self.documents + else None, "Instance": self.instance, "InstrumentationKey": self.instrumentation_key, "InvariantVersion": self.invariant_version, diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/__init__.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/__init__.py index 949c015..6c4d05e 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/__init__.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/__init__.py @@ -8,13 +8,19 @@ from azure_monitor.sdk.auto_collection.dependency_metrics import ( DependencyMetrics, ) +from azure_monitor.sdk.auto_collection.metrics_span_processor import ( + AzureMetricsSpanProcessor, +) from azure_monitor.sdk.auto_collection.performance_metrics import ( PerformanceMetrics, ) from azure_monitor.sdk.auto_collection.request_metrics import RequestMetrics +from azure_monitor.sdk.auto_collection.utils import AutoCollectionType __all__ = [ "AutoCollection", + "AutoCollectionType", + "AzureMetricsSpanProcessor", "DependencyMetrics", "RequestMetrics", "PerformanceMetrics", @@ -30,7 +36,15 @@ class AutoCollection: labels: Dictionary of labels """ - def __init__(self, meter: Meter, labels: Dict[str, str]): - self._performance_metrics = PerformanceMetrics(meter, labels) - self._dependency_metrics = DependencyMetrics(meter, labels) - self._request_metrics = RequestMetrics(meter, labels) + def __init__( + self, + meter: Meter, + labels: Dict[str, str], + span_processor: AzureMetricsSpanProcessor, + ): + col_type = AutoCollectionType.STANDARD_METRICS + self._performance_metrics = PerformanceMetrics(meter, labels, col_type) + self._dependency_metrics = DependencyMetrics( + meter, labels, span_processor + ) + self._request_metrics = RequestMetrics(meter, labels, span_processor) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/dependency_metrics.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/dependency_metrics.py index 5600612..47850c3 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/dependency_metrics.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/dependency_metrics.py @@ -1,59 +1,68 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -import threading import time from typing import Dict -import requests -from opentelemetry import context -from opentelemetry.metrics import Meter - -dependency_map = dict() -_dependency_lock = threading.Lock() -ORIGINAL_REQUEST = requests.Session.request +from opentelemetry.metrics import Meter, Observer +from azure_monitor.sdk.auto_collection.metrics_span_processor import ( + AzureMetricsSpanProcessor, +) -def dependency_patch(*args, **kwargs) -> None: - result = ORIGINAL_REQUEST(*args, **kwargs) - # Only collect request metric if sent from non-exporter thread - if context.get_value("suppress_instrumentation") is None: - # We don't want multiple threads updating this at once - with _dependency_lock: - count = dependency_map.get("count", 0) - dependency_map["count"] = count + 1 - return result +dependency_map = dict() class DependencyMetrics: - """Starts auto collection of dependency metrics, including - "Outgoing Requests per second" metric. + """Auto collection of dependency metrics, including + "Outgoing Requests per second", "Outgoing Request rate" and "Failed Outgoing Request rate" metrics. Args: meter: OpenTelemetry Meter labels: Dictionary of labels + span_processor: Azure Metrics Span Processor + collection_type: Standard or Live Metrics """ - def __init__(self, meter: Meter, labels: Dict[str, str]): + def __init__( + self, + meter: Meter, + labels: Dict[str, str], + span_processor: AzureMetricsSpanProcessor, + ): self._meter = meter self._labels = labels - # Patch requests - requests.Session.request = dependency_patch + self._span_processor = span_processor + + meter.register_observer( + callback=self._track_dependency_duration, + name="\\ApplicationInsights\\Dependency Call Duration", + description="Average Outgoing Requests duration", + unit="milliseconds", + value_type=int, + ) + meter.register_observer( + callback=self._track_failure_rate, + name="\\ApplicationInsights\\Dependency Calls Failed/Sec", + description="Failed Outgoing Requests per second", + unit="rps", + value_type=float, + ) meter.register_observer( callback=self._track_dependency_rate, name="\\ApplicationInsights\\Dependency Calls/Sec", description="Outgoing Requests per second", unit="rps", - value_type=int, + value_type=float, ) - def _track_dependency_rate(self, observer) -> None: + def _track_dependency_rate(self, observer: Observer) -> None: """ Track Dependency rate Calculated by obtaining the number of outgoing requests made using the requests library within an elapsed time and dividing that value over the elapsed time. """ - current_count = dependency_map.get("count", 0) + current_count = self._span_processor.dependency_count current_time = time.time() last_count = dependency_map.get("last_count", 0) last_time = dependency_map.get("last_time") @@ -66,12 +75,74 @@ def _track_dependency_rate(self, observer) -> None: interval_count = current_count - last_count result = interval_count / elapsed_seconds else: - result = 0 + result = 0.0 dependency_map["last_time"] = current_time dependency_map["last_count"] = current_count dependency_map["last_result"] = result + observer.observe(result, self._labels) + except ZeroDivisionError: + # If elapsed_seconds is 0, exporter call made too close to previous + # Return the previous result if this is the case + observer.observe(last_result, self._labels) + + def _track_dependency_duration(self, observer: Observer) -> None: + """ Track Dependency average duration + + Calculated by getting the time it takes to make an outgoing request + and dividing over the amount of outgoing requests over an elapsed time. + """ + last_average_duration = dependency_map.get("last_average_duration", 0) + interval_duration = ( + self._span_processor.dependency_duration + - dependency_map.get("last_duration", 0) + ) + interval_count = ( + self._span_processor.dependency_count + - dependency_map.get("last_count", 0) + ) + try: + result = interval_duration / interval_count + dependency_map[ + "last_count" + ] = self._span_processor.dependency_count + dependency_map["last_average_duration"] = result + dependency_map[ + "last_duration" + ] = self._span_processor.dependency_duration observer.observe(int(result), self._labels) + except ZeroDivisionError: + # If interval_count is 0, exporter call made too close to previous + # Return the previous result if this is the case + observer.observe(int(last_average_duration), self._labels) + + def _track_failure_rate(self, observer: Observer) -> None: + """ Track Failed Dependency rate + + Calculated by obtaining the number of failed outgoing requests made + using the requests library within an elapsed time and dividing + that value over the elapsed time. + """ + current_failed_count = self._span_processor.failed_dependency_count + current_time = time.time() + last_failed_count = dependency_map.get("last_failed_count", 0) + last_time = dependency_map.get("last_time") + last_result = dependency_map.get("last_result", 0) + + try: + # last_time is None the very first time this function is called + if last_time is not None: + elapsed_seconds = current_time - last_time + interval_failed_count = ( + current_failed_count - last_failed_count + ) + result = interval_failed_count / elapsed_seconds + else: + result = 0.0 + dependency_map["last_time"] = current_time + dependency_map["last_failed_count"] = current_failed_count + dependency_map["last_result"] = result + observer.observe(result, self._labels) except ZeroDivisionError: # If elapsed_seconds is 0, exporter call made too close to previous # Return the previous result if this is the case - observer.observe(int(last_result), self._labels) + observer.observe(last_result, self._labels) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py index 5b7f7a9..8c2cb46 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/__init__.py @@ -1,2 +1,53 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +# +from typing import Dict + +from opentelemetry.metrics import Meter + +from azure_monitor.sdk.auto_collection import AutoCollectionType +from azure_monitor.sdk.auto_collection.dependency_metrics import ( + DependencyMetrics, +) +from azure_monitor.sdk.auto_collection.live_metrics.manager import ( + LiveMetricsManager, +) +from azure_monitor.sdk.auto_collection.metrics_span_processor import ( + AzureMetricsSpanProcessor, +) +from azure_monitor.sdk.auto_collection.performance_metrics import ( + PerformanceMetrics, +) +from azure_monitor.sdk.auto_collection.request_metrics import RequestMetrics + +__all__ = ["LiveMetricsAutoCollection"] + + +class LiveMetricsAutoCollection: + """Starts auto collection of live metrics, including performance, + dependency and request metrics. + + Args: + meter: OpenTelemetry Meter + labels: Dictionary of labels + """ + + def __init__( + self, + meter: Meter, + labels: Dict[str, str], + span_processor: AzureMetricsSpanProcessor, + instrumentation_key: str, + ): + col_type = AutoCollectionType.LIVE_METRICS + self._performance_metrics = PerformanceMetrics(meter, labels, col_type) + self._dependency_metrics = DependencyMetrics( + meter, labels, span_processor + ) + self._request_metrics = RequestMetrics(meter, labels, span_processor) + self._manager = LiveMetricsManager( + meter, instrumentation_key, span_processor + ) + + def shutdown(self): + self._manager.shutdown() diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py index a8cc64d..7a265d7 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/exporter.py @@ -1,7 +1,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. # -import collections import logging import typing @@ -16,13 +15,15 @@ Envelope, LiveMetric, LiveMetricDocument, - LiveMetricDocumentProperty, LiveMetricEnvelope, ) from azure_monitor.sdk.auto_collection.live_metrics import utils from azure_monitor.sdk.auto_collection.live_metrics.sender import ( LiveMetricsSender, ) +from azure_monitor.sdk.auto_collection.metrics_span_processor import ( + AzureMetricsSpanProcessor, +) logger = logging.getLogger(__name__) @@ -36,14 +37,15 @@ class LiveMetricsExporter(MetricsExporter): Export data to Azure Live Metrics service and determine if user is subscribed. """ - def __init__(self, instrumentation_key): + def __init__( + self, + instrumentation_key: str, + span_processor: AzureMetricsSpanProcessor, + ): self._instrumentation_key = instrumentation_key + self._span_processor = span_processor self._sender = LiveMetricsSender(self._instrumentation_key) self.subscribed = True - self._document_envelopes = collections.deque() - - def add_document(self, envelope: Envelope): - self._document_envelopes.append(envelope) def export( self, metric_records: typing.Sequence[MetricRecord] @@ -91,23 +93,23 @@ def _get_live_metric_documents( self, ) -> typing.Sequence[LiveMetricDocument]: live_metric_documents = [] - while self._document_envelopes: - for envelope in self._document_envelopes.popleft(): - base_type = envelope.data.baseType - if base_type: - document = LiveMetricDocument( - __type=self._get_live_metric_type(base_type), - document_type=self._get_live_metric_document_type( - base_type - ), - properties=self._get_aggregated_properties(envelope), - version="1.0", - ) - live_metric_documents.append(document) - else: - logger.warning( - "Document type invalid; not sending live metric document" - ) + while self._span_processor.documents: + envelope = self._span_processor.documents.popleft() + base_type = envelope.data.base_type + if base_type: + document = LiveMetricDocument( + quickpulse_type=self._get_live_metric_type(base_type), + document_type=self._get_live_metric_document_type( + base_type + ), + properties=self._get_aggregated_properties(envelope), + version="1.0", + ) + live_metric_documents.append(document) + else: + logger.warning( + "Document type invalid; not sending live metric document" + ) return live_metric_documents @@ -145,25 +147,20 @@ def _get_live_metric_document_type(self, base_type: str) -> str: return "Availability" return "" - def _get_aggregated_properties( - self, envelope: Envelope - ) -> typing.List[LiveMetricDocumentProperty]: - - aggregated_properties = [] + def _get_aggregated_properties(self, envelope: Envelope) -> typing.Dict: + aggregated_properties = {} measurements = ( envelope.data.base_data.measurements - if envelope.data.base_data.measurements + if envelope.data.base_data and envelope.data.base_data.measurements else [] ) for key in measurements: - prop = LiveMetricDocumentProperty(key=key, value=measurements[key]) - aggregated_properties.append(prop) + aggregated_properties[key] = measurements[key] properties = ( envelope.data.base_data.properties - if envelope.data.base_data.properties + if envelope.data.base_data and envelope.data.base_data.properties else [] ) for key in properties: - prop = LiveMetricDocumentProperty(key=key, value=properties[key]) - aggregated_properties.append(prop) + aggregated_properties[key] = properties[key] return aggregated_properties diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py index 0e9c407..c93de77 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/manager.py @@ -5,6 +5,7 @@ import time from opentelemetry.context import attach, detach, set_value +from opentelemetry.sdk.metrics import Meter from opentelemetry.sdk.metrics.export import MetricsExportResult from azure_monitor.sdk.auto_collection.live_metrics import utils @@ -14,6 +15,9 @@ from azure_monitor.sdk.auto_collection.live_metrics.sender import ( LiveMetricsSender, ) +from azure_monitor.sdk.auto_collection.metrics_span_processor import ( + AzureMetricsSpanProcessor, +) # Interval for failures threshold reached in seconds FALLBACK_INTERVAL = 60.0 @@ -34,14 +38,22 @@ class LiveMetricsManager(threading.Thread): daemon = True - def __init__(self, meter, instrumentation_key): + def __init__( + self, + meter: Meter, + instrumentation_key: str, + span_processor: AzureMetricsSpanProcessor, + ): super().__init__() self.thread_event = threading.Event() self.interval = MAIN_INTERVAL self._instrumentation_key = instrumentation_key self._is_user_subscribed = False self._meter = meter - self._exporter = LiveMetricsExporter(self._instrumentation_key) + self._span_processor = span_processor + self._exporter = LiveMetricsExporter( + self._instrumentation_key, self._span_processor + ) self._post = None self._ping = LiveMetricsPing(self._instrumentation_key) self.start() @@ -57,12 +69,14 @@ def check_if_user_is_subscribed(self): # Switch to Post self._ping.shutdown() self._ping = None + self._span_processor.is_collecting_documents = True self._post = LiveMetricsPost( self._meter, self._exporter, self._instrumentation_key ) if self._post: if not self._post.is_user_subscribed: # Switch to Ping + self._span_processor.is_collecting_documents = False self._post.shutdown() self._post = None self._ping = LiveMetricsPing(self._instrumentation_key) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py index 248db5f..27f39e3 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/sender.py @@ -3,6 +3,7 @@ # import json import logging +import time import requests @@ -41,7 +42,7 @@ def _send_request(self, data: str, request_type: str) -> requests.Response: "Expect": "100-continue", "Content-Type": "application/json; charset=utf-8", utils.LIVE_METRICS_TRANSMISSION_TIME_HEADER: str( - utils.calculate_ticks_since_epoch() + round(time.time()) * 1000 ), }, ) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py index 724467d..cfba984 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/live_metrics/utils.py @@ -1,8 +1,8 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. # +import time import uuid -from datetime import datetime from azure_monitor.protocol import LiveMetricEnvelope from azure_monitor.utils import azure_monitor_context @@ -22,13 +22,7 @@ def create_metric_envelope(instrumentation_key: str): machine_name=azure_monitor_context.get("ai.device.id"), metrics=None, stream_id=STREAM_ID, - timestamp="/Date({0})/".format(str(calculate_ticks_since_epoch())), + timestamp="/Date({0})/".format(str(int(time.time()) * 1000)), version=azure_monitor_context.get("ai.internal.sdkVersion"), ) return envelope - - -def calculate_ticks_since_epoch(): - return round( - (datetime.utcnow() - datetime(1, 1, 1)).total_seconds() * 10000000 - ) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/metrics_span_processor.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/metrics_span_processor.py new file mode 100644 index 0000000..1d007dd --- /dev/null +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/metrics_span_processor.py @@ -0,0 +1,64 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import collections +import logging + +from opentelemetry.sdk.trace import Span, SpanProcessor +from opentelemetry.trace import SpanKind + +from azure_monitor.export import convert_span_to_envelope + +logger = logging.getLogger(__name__) + + +class AzureMetricsSpanProcessor(SpanProcessor): + """AzureMetricsSpanProcessor is an implementation of `SpanProcessor` used + to generate Azure specific metrics, including dependencies/requests rate, average duration + and failed dependencies/requests. + """ + + def __init__(self): + self.is_collecting_documents = False + self.documents = collections.deque() + self.request_count = 0 + self.dependency_count = 0 + self.failed_request_count = 0 + self.failed_dependency_count = 0 + self.request_duration = 0 + self.dependency_duration = 0 + + def on_start(self, span: Span) -> None: + pass + + def on_end(self, span: Span) -> None: + try: + if span.kind == SpanKind.SERVER: + self.request_count = self.request_count + 1 + duration = ( + span.end_time - span.start_time + ) / 1000000 # Convert to milliseconds + self.request_duration = self.request_duration + duration + if not span.status.is_ok: + self.failed_request_count = self.failed_request_count + 1 + if self.is_collecting_documents: + self.documents.append(convert_span_to_envelope(span)) + + elif span.kind == SpanKind.CLIENT: + self.dependency_count = self.dependency_count + 1 + duration = ( + span.end_time - span.start_time + ) / 1000000 # Convert to milliseconds + self.dependency_duration = self.dependency_duration + duration + if not span.status.is_ok: + self.failed_dependency_count = ( + self.failed_dependency_count + 1 + ) + if self.is_collecting_documents: + self.documents.append(convert_span_to_envelope(span)) + + # pylint: disable=broad-except + except Exception: + logger.exception("Exception while processing Span.") + + def shutdown(self) -> None: + pass diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/performance_metrics.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/performance_metrics.py index 9e1927a..59c335b 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/performance_metrics.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/performance_metrics.py @@ -4,7 +4,9 @@ from typing import Dict import psutil -from opentelemetry.metrics import Meter +from opentelemetry.metrics import Meter, Observer + +from azure_monitor.sdk.auto_collection.utils import AutoCollectionType logger = logging.getLogger(__name__) PROCESS = psutil.Process() @@ -19,42 +21,58 @@ class PerformanceMetrics: Args: meter: OpenTelemetry Meter labels: Dictionary of labels + collection_type: Standard or Live Metrics """ - def __init__(self, meter: Meter, labels: Dict[str, str]): + def __init__( + self, + meter: Meter, + labels: Dict[str, str], + collection_type: AutoCollectionType, + ): self._meter = meter self._labels = labels - # Create performance metrics - meter.register_observer( + + self._meter.register_observer( callback=self._track_cpu, name="\\Processor(_Total)\\% Processor Time", description="Processor time as a percentage", unit="percentage", value_type=float, ) - meter.register_observer( - callback=self._track_memory, - name="\\Memory\\Available Bytes", - description="Amount of available memory in bytes", - unit="byte", - value_type=int, - ) - meter.register_observer( - callback=self._track_process_cpu, - name="\\Process(??APP_WIN32_PROC??)\\% Processor Time", - description="Process CPU usage as a percentage", - unit="percentage", - value_type=float, - ) - meter.register_observer( - callback=self._track_process_memory, - name="\\Process(??APP_WIN32_PROC??)\\Private Bytes", - description="Amount of memory process has used in bytes", - unit="byte", - value_type=int, - ) - def _track_cpu(self, observer) -> None: + if collection_type == AutoCollectionType.STANDARD_METRICS: + self._meter.register_observer( + callback=self._track_memory, + name="\\Memory\\Available Bytes", + description="Amount of available memory in bytes", + unit="byte", + value_type=int, + ) + self._meter.register_observer( + callback=self._track_process_cpu, + name="\\Process(??APP_WIN32_PROC??)\\% Processor Time", + description="Process CPU usage as a percentage", + unit="percentage", + value_type=float, + ) + self._meter.register_observer( + callback=self._track_process_memory, + name="\\Process(??APP_WIN32_PROC??)\\Private Bytes", + description="Amount of memory process has used in bytes", + unit="byte", + value_type=int, + ) + if collection_type == AutoCollectionType.LIVE_METRICS: + self._meter.register_observer( + callback=self._track_commited_memory, + name="\\Memory\\Committed Bytes", + description="Amount of commited memory in bytes", + unit="byte", + value_type=int, + ) + + def _track_cpu(self, observer: Observer) -> None: """ Track CPU time Processor time is defined as a float representing the current system @@ -65,7 +83,7 @@ def _track_cpu(self, observer) -> None: cpu_times_percent = psutil.cpu_times_percent() observer.observe(100.0 - cpu_times_percent.idle, self._labels) - def _track_memory(self, observer) -> None: + def _track_memory(self, observer: Observer) -> None: """ Track Memory Available memory is defined as memory that can be given instantly to @@ -73,7 +91,7 @@ def _track_memory(self, observer) -> None: """ observer.observe(psutil.virtual_memory().available, self._labels) - def _track_process_cpu(self, observer) -> None: + def _track_process_cpu(self, observer: Observer) -> None: """ Track Process CPU time Returns a derived gauge for the CPU usage for the current process. @@ -88,7 +106,7 @@ def _track_process_cpu(self, observer) -> None: except Exception: # pylint: disable=broad-except logger.exception("Error handling get process cpu usage.") - def _track_process_memory(self, observer) -> None: + def _track_process_memory(self, observer: Observer) -> None: """ Track Memory Available memory is defined as memory that can be given instantly to @@ -98,3 +116,13 @@ def _track_process_memory(self, observer) -> None: observer.observe(PROCESS.memory_info().rss, self._labels) except Exception: # pylint: disable=broad-except logger.exception("Error handling get process private bytes.") + + def _track_commited_memory(self, observer: Observer) -> None: + """ Track Commited Memory + + Available commited memory is defined as total memory minus available memory. + """ + observer.observe( + psutil.virtual_memory().total - psutil.virtual_memory().available, + self._labels, + ) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/request_metrics.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/request_metrics.py index 8647ec6..e1a3809 100644 --- a/azure_monitor/src/azure_monitor/sdk/auto_collection/request_metrics.py +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/request_metrics.py @@ -1,74 +1,48 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. import logging -import threading import time -from http.server import HTTPServer from typing import Dict -from opentelemetry.metrics import Meter +from opentelemetry.metrics import Meter, Observer -logger = logging.getLogger(__name__) +from azure_monitor.sdk.auto_collection.metrics_span_processor import ( + AzureMetricsSpanProcessor, +) -_requests_lock = threading.Lock() +logger = logging.getLogger(__name__) requests_map = dict() -ORIGINAL_CONSTRUCTOR = HTTPServer.__init__ - - -def request_patch(func): - def wrapper(self=None): - start_time = time.time() - func(self) - end_time = time.time() - - with _requests_lock: - # Update Count - count = requests_map.get("count", 0) - requests_map["count"] = count + 1 - # Update duration - duration = requests_map.get("duration", 0) - requests_map["duration"] = duration + (end_time - start_time) - - return wrapper - - -def server_patch(*args, **kwargs): - if len(args) >= 3: - handler = args[2] - if handler: - # Patch the handler methods if they exist - if "do_DELETE" in dir(handler): - handler.do_DELETE = request_patch(handler.do_DELETE) - if "do_GET" in dir(handler): - handler.do_GET = request_patch(handler.do_GET) - if "do_HEAD" in dir(handler): - handler.do_HEAD = request_patch(handler.do_HEAD) - if "do_OPTIONS" in dir(handler): - handler.do_OPTIONS = request_patch(handler.do_OPTIONS) - if "do_POST" in dir(handler): - handler.do_POST = request_patch(handler.do_POST) - if "do_PUT" in dir(handler): - handler.do_PUT = request_patch(handler.do_PUT) - result = ORIGINAL_CONSTRUCTOR(*args, **kwargs) - return result class RequestMetrics: """Starts auto collection of request metrics, including - "Incoming Requests Average Execution Time" and - "Incoming Requests Average Execution Rate" metrics. + "Incoming Requests Average Execution Time", + "Incoming Requests Rate" and "Failed Incoming Requests Rate" metrics. Args: meter: OpenTelemetry Meter labels: Dictionary of labels + span_processor: Azure Metrics Span Processor + collection_type: Standard or Live Metrics """ - def __init__(self, meter: Meter, labels: Dict[str, str]): + def __init__( + self, + meter: Meter, + labels: Dict[str, str], + span_processor: AzureMetricsSpanProcessor, + ): self._meter = meter self._labels = labels - # Patch the HTTPServer handler to track request information - HTTPServer.__init__ = server_patch + self._span_processor = span_processor + meter.register_observer( + callback=self._track_request_failed_rate, + name="\\ApplicationInsights\\Requests Failed/Sec", + description="Incoming Requests Failed Rate", + unit="rps", + value_type=float, + ) meter.register_observer( callback=self._track_request_duration, name="\\ASP.NET Applications(??APP_W3SVC_PROC??)\\Request Execution Time", @@ -76,40 +50,42 @@ def __init__(self, meter: Meter, labels: Dict[str, str]): unit="milliseconds", value_type=int, ) - meter.register_observer( callback=self._track_request_rate, name="\\ASP.NET Applications(??APP_W3SVC_PROC??)\\Requests/Sec", - description="Incoming Requests Average Execution Rate", + description="Incoming Requests Rate", unit="rps", - value_type=int, + value_type=float, ) - def _track_request_duration(self, observer) -> None: + def _track_request_duration(self, observer: Observer) -> None: """ Track Request execution time Calculated by getting the time it takes to make an incoming request and dividing over the amount of incoming requests over an elapsed time. """ last_average_duration = requests_map.get("last_average_duration", 0) - interval_duration = requests_map.get("duration", 0) - requests_map.get( - "last_duration", 0 + interval_duration = ( + self._span_processor.request_duration + - requests_map.get("last_duration", 0) ) - interval_count = requests_map.get("count", 0) - requests_map.get( + interval_count = self._span_processor.request_count - requests_map.get( "last_count", 0 ) try: result = interval_duration / interval_count + requests_map["last_count"] = self._span_processor.request_count requests_map["last_average_duration"] = result - requests_map["last_duration"] = requests_map.get("duration", 0) - # Convert to milliseconds - observer.observe(int(result * 1000.0), self._labels) + requests_map[ + "last_duration" + ] = self._span_processor.request_duration + observer.observe(int(result), self._labels) except ZeroDivisionError: # If interval_count is 0, exporter call made too close to previous # Return the previous result if this is the case - observer.observe(int(last_average_duration * 1000.0), self._labels) + observer.observe(int(last_average_duration), self._labels) - def _track_request_rate(self, observer) -> None: + def _track_request_rate(self, observer: Observer) -> None: """ Track Request execution rate Calculated by obtaining by getting the number of incoming requests @@ -124,17 +100,51 @@ def _track_request_rate(self, observer) -> None: # last_rate_time is None the first time this function is called if last_time is not None: interval_time = current_time - requests_map.get("last_time", 0) - interval_count = requests_map.get( - "count", 0 - ) - requests_map.get("last_count", 0) + interval_count = ( + self._span_processor.request_count + - requests_map.get("last_count", 0) + ) result = interval_count / interval_time else: - result = 0 + result = 0.0 requests_map["last_time"] = current_time - requests_map["last_count"] = requests_map.get("count", 0) + requests_map["last_count"] = self._span_processor.request_count requests_map["last_rate"] = result - observer.observe(int(result), self._labels) + observer.observe(result, self._labels) + except ZeroDivisionError: + # If elapsed_seconds is 0, exporter call made too close to previous + # Return the previous result if this is the case + observer.observe(last_rate, self._labels) + + def _track_request_failed_rate(self, observer: Observer) -> None: + """ Track Request failed execution rate + + Calculated by obtaining by getting the number of failed incoming requests + made to an HTTPServer within an elapsed time and dividing that value + over the elapsed time. + """ + current_time = time.time() + last_rate = requests_map.get("last_rate", 0) + last_time = requests_map.get("last_time") + + try: + # last_rate_time is None the first time this function is called + if last_time is not None: + interval_time = current_time - requests_map.get("last_time", 0) + interval_count = ( + self._span_processor.failed_request_count + - requests_map.get("last_failed_count", 0) + ) + result = interval_count / interval_time + else: + result = 0.0 + requests_map["last_time"] = current_time + requests_map[ + "last_failed_count" + ] = self._span_processor.failed_request_count + requests_map["last_rate"] = result + observer.observe(result, self._labels) except ZeroDivisionError: # If elapsed_seconds is 0, exporter call made too close to previous # Return the previous result if this is the case - observer.observe(int(last_rate), self._labels) + observer.observe(last_rate, self._labels) diff --git a/azure_monitor/src/azure_monitor/sdk/auto_collection/utils.py b/azure_monitor/src/azure_monitor/sdk/auto_collection/utils.py new file mode 100644 index 0000000..acbc811 --- /dev/null +++ b/azure_monitor/src/azure_monitor/sdk/auto_collection/utils.py @@ -0,0 +1,11 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +from enum import Enum + + +class AutoCollectionType(Enum): + """Automatic collection of metrics type + """ + + STANDARD_METRICS = 0 + LIVE_METRICS = 1 diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py b/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py index f37e8ec..17cbb07 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_exporter.py @@ -8,12 +8,23 @@ from opentelemetry import metrics from opentelemetry.sdk.metrics import Counter, MeterProvider from opentelemetry.sdk.metrics.export import MetricRecord, MetricsExportResult -from opentelemetry.sdk.metrics.export.aggregate import CounterAggregator +from opentelemetry.sdk.metrics.export.aggregate import ( + CounterAggregator, + ObserverAggregator, +) -from azure_monitor.protocol import Envelope +from azure_monitor.protocol import ( + Data, + Envelope, + LiveMetricEnvelope, + RemoteDependency, +) from azure_monitor.sdk.auto_collection.live_metrics.exporter import ( LiveMetricsExporter, ) +from azure_monitor.sdk.auto_collection.metrics_span_processor import ( + AzureMetricsSpanProcessor, +) def throw(exc_type, *args, **kwargs): @@ -33,34 +44,37 @@ def setUpClass(cls): cls._test_metric = cls._meter.create_metric( "testname", "testdesc", "unit", int, Counter, ["environment"] ) + cls._test_obs = cls._meter.register_observer( + lambda x: x, + "testname", + "testdesc", + "unit", + int, + Counter, + ["environment"], + ) cls._test_labels = tuple({"environment": "staging"}.items()) + cls._span_processor = AzureMetricsSpanProcessor() def test_constructor(self): """Test the constructor.""" exporter = LiveMetricsExporter( - instrumentation_key=self._instrumentation_key + instrumentation_key=self._instrumentation_key, + span_processor=self._span_processor, ) self.assertEqual(exporter.subscribed, True) self.assertEqual( exporter._instrumentation_key, self._instrumentation_key ) - def test_add_document(self): - """Test adding a document.""" - exporter = LiveMetricsExporter( - instrumentation_key=self._instrumentation_key - ) - envelope = Envelope() - exporter.add_document(envelope) - self.assertEqual(exporter._document_envelopes.pop(), envelope) - def test_export(self): """Test export.""" record = MetricRecord( CounterAggregator(), self._test_labels, self._test_metric ) exporter = LiveMetricsExporter( - instrumentation_key=self._instrumentation_key + instrumentation_key=self._instrumentation_key, + span_processor=self._span_processor, ) with mock.patch( "azure_monitor.sdk.auto_collection.live_metrics.sender.LiveMetricsSender.post" @@ -76,7 +90,8 @@ def test_export_failed(self): CounterAggregator(), self._test_labels, self._test_metric ) exporter = LiveMetricsExporter( - instrumentation_key=self._instrumentation_key + instrumentation_key=self._instrumentation_key, + span_processor=self._span_processor, ) with mock.patch( "azure_monitor.sdk.auto_collection.live_metrics.sender.LiveMetricsSender.post" @@ -92,7 +107,8 @@ def test_export_exception(self): CounterAggregator(), self._test_labels, self._test_metric ) exporter = LiveMetricsExporter( - instrumentation_key=self._instrumentation_key + instrumentation_key=self._instrumentation_key, + span_processor=self._span_processor, ) with mock.patch( "azure_monitor.sdk.auto_collection.live_metrics.sender.LiveMetricsSender.post", @@ -100,3 +116,101 @@ def test_export_exception(self): ): result = exporter.export([record]) self.assertEqual(result, MetricsExportResult.FAILURE) + + def test_live_metric_envelope_counter(self): + aggregator = ObserverAggregator() + aggregator.update(123) + aggregator.take_checkpoint() + record = MetricRecord(aggregator, self._test_labels, self._test_obs) + exporter = LiveMetricsExporter( + instrumentation_key=self._instrumentation_key, + span_processor=self._span_processor, + ) + + envelope = exporter._metric_to_live_metrics_envelope([record]) + self.assertIsInstance(envelope, LiveMetricEnvelope) + self.assertEqual( + envelope.instrumentation_key, + "99c42f65-1656-4c41-afde-bd86b709a4a7", + ) + self.assertEqual(envelope.documents, []) + self.assertEqual(envelope.metrics[0].name, "testname") + self.assertEqual(envelope.metrics[0].value, 123) + self.assertEqual(envelope.metrics[0].weight, 1) + + def test_live_metric_envelope_observer(self): + aggregator = CounterAggregator() + aggregator.update(123) + aggregator.take_checkpoint() + record = MetricRecord(aggregator, self._test_labels, self._test_metric) + exporter = LiveMetricsExporter( + instrumentation_key=self._instrumentation_key, + span_processor=self._span_processor, + ) + + envelope = exporter._metric_to_live_metrics_envelope([record]) + self.assertIsInstance(envelope, LiveMetricEnvelope) + self.assertEqual(envelope.documents, []) + self.assertEqual(envelope.metrics[0].name, "testname") + self.assertEqual(envelope.metrics[0].value, 123) + self.assertEqual(envelope.metrics[0].weight, 1) + + def test_live_metric_envelope_documents(self): + aggregator = CounterAggregator() + aggregator.update(123) + aggregator.take_checkpoint() + record = MetricRecord(aggregator, self._test_labels, self._test_metric) + exporter = LiveMetricsExporter( + instrumentation_key=self._instrumentation_key, + span_processor=self._span_processor, + ) + request_data = RemoteDependency( + name="testName", + id="", + result_code="testResultCode", + duration="testDuration", + success=True, + properties={}, + measurements={}, + ) + request_data.properties["test_property1"] = "test_property1Value" + request_data.properties["test_property2"] = "test_property2Value" + request_data.measurements[ + "test_measurement1" + ] = "test_measurement1Value" + request_data.measurements[ + "test_measurement2" + ] = "test_measurement2Value" + test_envelope = Envelope( + data=Data(base_type="RemoteDependencyData", base_data=request_data) + ) + self._span_processor.documents.append(test_envelope) + envelope = exporter._metric_to_live_metrics_envelope([record]) + self.assertIsInstance(envelope, LiveMetricEnvelope) + self.assertEqual(len(envelope.documents), 1) + self.assertEqual( + envelope.documents[0].quickpulse_type, + "DependencyTelemetryDocument", + ) + self.assertEqual( + envelope.documents[0].document_type, "RemoteDependency" + ) + self.assertEqual(envelope.documents[0].version, "1.0") + self.assertEqual(envelope.documents[0].operation_id, "") + self.assertEqual(len(envelope.documents[0].properties), 4) + self.assertEqual( + envelope.documents[0].properties["test_measurement1"], + "test_measurement1Value", + ) + self.assertEqual( + envelope.documents[0].properties["test_measurement2"], + "test_measurement2Value", + ) + self.assertEqual( + envelope.documents[0].properties["test_property1"], + "test_property1Value", + ) + self.assertEqual( + envelope.documents[0].properties["test_property2"], + "test_property2Value", + ) diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_live_metrics_auto_collection.py b/azure_monitor/tests/auto_collection/live_metrics/test_live_metrics_auto_collection.py new file mode 100644 index 0000000..e94bcee --- /dev/null +++ b/azure_monitor/tests/auto_collection/live_metrics/test_live_metrics_auto_collection.py @@ -0,0 +1,46 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import unittest + +from opentelemetry import metrics +from opentelemetry.sdk.metrics import MeterProvider + +from azure_monitor.sdk.auto_collection import AzureMetricsSpanProcessor +from azure_monitor.sdk.auto_collection.live_metrics import ( + LiveMetricsAutoCollection, +) + + +# pylint: disable=protected-access +class TestLiveMetricsAutoCollection(unittest.TestCase): + @classmethod + def setUpClass(cls): + metrics.set_meter_provider(MeterProvider()) + cls._meter = metrics.get_meter(__name__) + cls._test_labels = tuple({"environment": "staging"}.items()) + cls._span_processor = AzureMetricsSpanProcessor() + + @classmethod + def tearDownClass(cls): + metrics._METER_PROVIDER = None + + def setUp(self): + self._auto_collection = LiveMetricsAutoCollection( + meter=self._meter, + labels=self._test_labels, + span_processor=self._span_processor, + instrumentation_key="TEST", + ) + + def tearDown(self): + self._auto_collection.shutdown() + + def test_constructor(self): + """Test the constructor.""" + self.assertIsNotNone(self._auto_collection._performance_metrics) + self.assertIsNotNone(self._auto_collection._dependency_metrics) + self.assertIsNotNone(self._auto_collection._request_metrics) + self.assertIsNotNone(self._auto_collection._manager) + # Check observers + self.assertEqual(len(self._meter.observers), 8) diff --git a/azure_monitor/tests/auto_collection/live_metrics/test_manager.py b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py index 4c4cab7..7b9dc1d 100644 --- a/azure_monitor/tests/auto_collection/live_metrics/test_manager.py +++ b/azure_monitor/tests/auto_collection/live_metrics/test_manager.py @@ -16,6 +16,9 @@ LiveMetricsPing, LiveMetricsPost, ) +from azure_monitor.sdk.auto_collection.metrics_span_processor import ( + AzureMetricsSpanProcessor, +) # pylint: disable=protected-access @@ -33,6 +36,7 @@ def setUpClass(cls): cls._manager = None cls._ping = None cls._post = None + cls._span_processor = AzureMetricsSpanProcessor() @classmethod def tearDownClass(cls): @@ -55,6 +59,7 @@ def test_constructor(self): self._manager = LiveMetricsManager( meter=self._meter, instrumentation_key=self._instrumentation_key, + span_processor=self._span_processor, ) self.assertFalse(self._manager._is_user_subscribed) self.assertEqual( @@ -72,16 +77,23 @@ def test_switch(self): self._manager = LiveMetricsManager( meter=self._meter, instrumentation_key=self._instrumentation_key, + span_processor=self._span_processor, ) self._manager.interval = 60 time.sleep(1) self._manager.check_if_user_is_subscribed() self.assertIsNone(self._manager._ping) self.assertIsNotNone(self._manager._post) + self.assertEqual( + self._manager._span_processor.is_collecting_documents, True + ) self._manager._post.is_user_subscribed = False self._manager.check_if_user_is_subscribed() self.assertIsNone(self._manager._post) self.assertIsNotNone(self._manager._ping) + self.assertEqual( + self._manager._span_processor.is_collecting_documents, False + ) def test_ping_ok(self): """Test ping send requests to Live Metrics service.""" @@ -115,7 +127,7 @@ def test_ping_error(self): self._ping = LiveMetricsPing( instrumentation_key=self._instrumentation_key ) - self._ping.last_request_success_time = time.time() - 21 + self._ping.last_request_success_time = time.time() - 60 self._ping.ping() self.assertFalse(self._ping.last_send_succeeded) self.assertEqual(self._ping.interval, 60) @@ -127,7 +139,10 @@ def test_post_ok(self): 200, None, {"x-ms-qps-subscribed": "false"} ) self._post = LiveMetricsPost( - exporter=LiveMetricsExporter(self._instrumentation_key), + exporter=LiveMetricsExporter( + self._instrumentation_key, + span_processor=self._span_processor, + ), meter=self._meter, instrumentation_key=self._instrumentation_key, ) @@ -144,7 +159,10 @@ def test_post_subscribed(self): 200, None, {"x-ms-qps-subscribed": "true"} ) self._post = LiveMetricsPost( - exporter=LiveMetricsExporter(self._instrumentation_key), + exporter=LiveMetricsExporter( + self._instrumentation_key, + span_processor=self._span_processor, + ), meter=self._meter, instrumentation_key=self._instrumentation_key, ) @@ -156,7 +174,10 @@ def test_post_error(self): with mock.patch("requests.post") as request: request.return_value = MockResponse(400, None, {}) self._post = LiveMetricsPost( - exporter=LiveMetricsExporter(self._instrumentation_key), + exporter=LiveMetricsExporter( + self._instrumentation_key, + span_processor=self._span_processor, + ), meter=self._meter, instrumentation_key=self._instrumentation_key, ) diff --git a/azure_monitor/tests/auto_collection/test_auto_collection.py b/azure_monitor/tests/auto_collection/test_auto_collection.py index c965d4b..25c1c81 100644 --- a/azure_monitor/tests/auto_collection/test_auto_collection.py +++ b/azure_monitor/tests/auto_collection/test_auto_collection.py @@ -7,7 +7,10 @@ from opentelemetry import metrics from opentelemetry.sdk.metrics import MeterProvider -from azure_monitor.sdk.auto_collection import AutoCollection +from azure_monitor.sdk.auto_collection import ( + AutoCollection, + AzureMetricsSpanProcessor, +) # pylint: disable=protected-access @@ -17,6 +20,7 @@ def setUpClass(cls): metrics.set_meter_provider(MeterProvider()) cls._meter = metrics.get_meter(__name__) cls._test_labels = tuple({"environment": "staging"}.items()) + cls._span_processor = AzureMetricsSpanProcessor() @classmethod def tearDownClass(cls): @@ -35,7 +39,12 @@ def test_constructor( self, mock_performance, mock_dependencies, mock_requests ): """Test the constructor.""" - AutoCollection(meter=self._meter, labels=self._test_labels) + + AutoCollection( + meter=self._meter, + labels=self._test_labels, + span_processor=self._span_processor, + ) self.assertEqual(mock_performance.called, True) self.assertEqual(mock_dependencies.called, True) self.assertEqual(mock_requests.called, True) diff --git a/azure_monitor/tests/auto_collection/test_dependency_metrics.py b/azure_monitor/tests/auto_collection/test_dependency_metrics.py index df0a8a1..7ea3413 100644 --- a/azure_monitor/tests/auto_collection/test_dependency_metrics.py +++ b/azure_monitor/tests/auto_collection/test_dependency_metrics.py @@ -2,17 +2,15 @@ # Licensed under the MIT License. import unittest -from http.server import HTTPServer from unittest import mock -import requests from opentelemetry import metrics from opentelemetry.sdk.metrics import MeterProvider, Observer from azure_monitor.sdk.auto_collection import dependency_metrics - -ORIGINAL_FUNCTION = requests.Session.request -ORIGINAL_CONS = HTTPServer.__init__ +from azure_monitor.sdk.auto_collection.metrics_span_processor import ( + AzureMetricsSpanProcessor, +) # pylint: disable=protected-access @@ -22,50 +20,67 @@ def setUpClass(cls): metrics.set_meter_provider(MeterProvider()) cls._meter = metrics.get_meter(__name__) cls._test_labels = {"environment": "staging"} + cls._span_processor = AzureMetricsSpanProcessor() @classmethod def tearDown(cls): metrics._METER_PROVIDER = None - requests.Session.request = ORIGINAL_FUNCTION - dependency_metrics.ORIGINAL_CONSTRUCTOR = ORIGINAL_CONS def setUp(self): dependency_metrics.dependency_map.clear() - requests.Session.request = ORIGINAL_FUNCTION - dependency_metrics.ORIGINAL_CONSTRUCTOR = ORIGINAL_CONS def test_constructor(self): mock_meter = mock.Mock() metrics_collector = dependency_metrics.DependencyMetrics( - meter=mock_meter, labels=self._test_labels + meter=mock_meter, + labels=self._test_labels, + span_processor=self._span_processor, ) self.assertEqual(metrics_collector._meter, mock_meter) self.assertEqual(metrics_collector._labels, self._test_labels) - self.assertEqual(mock_meter.register_observer.call_count, 1) - mock_meter.register_observer.assert_called_with( + self.assertEqual(mock_meter.register_observer.call_count, 3) + + create_metric_calls = mock_meter.register_observer.call_args_list + create_metric_calls[0].assert_called_with( + callback=metrics_collector._track_dependency_duration, + name="\\ApplicationInsights\\Dependency Call Duration", + description="Average Outgoing Requests duration", + unit="milliseconds", + value_type=int, + ) + create_metric_calls[1].assert_called_with( + callback=metrics_collector._track_failure_rate, + name="\\ApplicationInsights\\Dependency Calls Failed/Sec", + description="Failed Outgoing Requests per second", + unit="rps", + value_type=float, + ) + create_metric_calls[2].assert_called_with( callback=metrics_collector._track_dependency_rate, name="\\ApplicationInsights\\Dependency Calls/Sec", description="Outgoing Requests per second", unit="rps", - value_type=int, + value_type=float, ) @mock.patch("azure_monitor.sdk.auto_collection.dependency_metrics.time") def test_track_dependency_rate(self, time_mock): time_mock.time.return_value = 100 metrics_collector = dependency_metrics.DependencyMetrics( - meter=self._meter, labels=self._test_labels + meter=self._meter, + labels=self._test_labels, + span_processor=self._span_processor, ) obs = Observer( callback=metrics_collector._track_dependency_rate, name="\\ApplicationInsights\\Dependency Calls/Sec", description="Outgoing Requests per second", unit="rps", - value_type=int, + value_type=float, meter=self._meter, ) - dependency_metrics.dependency_map["last_time"] = 98 - dependency_metrics.dependency_map["count"] = 4 + dependency_metrics.dependency_map["last_time"] = 98.0 + self._span_processor.dependency_count = 4 metrics_collector._track_dependency_rate(obs) self.assertEqual( obs.aggregators[tuple(self._test_labels.items())].current, 2 @@ -75,7 +90,9 @@ def test_track_dependency_rate(self, time_mock): def test_track_dependency_rate_time_none(self, time_mock): time_mock.time.return_value = 100 metrics_collector = dependency_metrics.DependencyMetrics( - meter=self._meter, labels=self._test_labels + meter=self._meter, + labels=self._test_labels, + span_processor=self._span_processor, ) dependency_metrics.dependency_map["last_time"] = None obs = Observer( @@ -83,51 +100,145 @@ def test_track_dependency_rate_time_none(self, time_mock): name="\\ApplicationInsights\\Dependency Calls/Sec", description="Outgoing Requests per second", unit="rps", - value_type=int, + value_type=float, meter=self._meter, ) metrics_collector._track_dependency_rate(obs) self.assertEqual( - obs.aggregators[tuple(self._test_labels.items())].current, 0 + obs.aggregators[tuple(self._test_labels.items())].current, 0.0 ) @mock.patch("azure_monitor.sdk.auto_collection.dependency_metrics.time") def test_track_dependency_rate_error(self, time_mock): time_mock.time.return_value = 100 metrics_collector = dependency_metrics.DependencyMetrics( - meter=self._meter, labels=self._test_labels + meter=self._meter, + labels=self._test_labels, + span_processor=self._span_processor, ) dependency_metrics.dependency_map["last_time"] = 100 - dependency_metrics.dependency_map["last_result"] = 5 + dependency_metrics.dependency_map["last_result"] = 5.0 obs = Observer( callback=metrics_collector._track_dependency_rate, name="\\ApplicationInsights\\Dependency Calls/Sec", description="Outgoing Requests per second", unit="rps", - value_type=int, + value_type=float, meter=self._meter, ) metrics_collector._track_dependency_rate(obs) self.assertEqual( - obs.aggregators[tuple(self._test_labels.items())].current, 5 - ) - - @mock.patch( - "azure_monitor.sdk.auto_collection.dependency_metrics.ORIGINAL_REQUEST" - ) - def test_dependency_patch(self, request_mock): - session = requests.Session() - dependency_metrics.dependency_patch(session) - self.assertEqual(dependency_metrics.dependency_map["count"], 1) - request_mock.assert_called_with(session) - - @mock.patch( - "azure_monitor.sdk.auto_collection.dependency_metrics.ORIGINAL_REQUEST" - ) - @mock.patch("azure_monitor.sdk.auto_collection.dependency_metrics.context") - def test_dependency_patch_suppress(self, context_mock, request_mock): - context_mock.get_value.return_value = {} - session = requests.Session() - dependency_metrics.dependency_patch(session) - self.assertEqual(dependency_metrics.dependency_map.get("count"), None) - request_mock.assert_called_with(session) + obs.aggregators[tuple(self._test_labels.items())].current, 5.0 + ) + + @mock.patch("azure_monitor.sdk.auto_collection.dependency_metrics.time") + def test_track_failed_dependency_rate(self, time_mock): + time_mock.time.return_value = 100 + metrics_collector = dependency_metrics.DependencyMetrics( + meter=self._meter, + labels=self._test_labels, + span_processor=self._span_processor, + ) + obs = Observer( + callback=metrics_collector._track_failure_rate, + name="test", + description="test", + unit="test", + value_type=float, + meter=self._meter, + ) + dependency_metrics.dependency_map["last_time"] = 98 + self._span_processor.failed_dependency_count = 4 + metrics_collector._track_failure_rate(obs) + self.assertEqual( + obs.aggregators[tuple(self._test_labels.items())].current, 2.0 + ) + + @mock.patch("azure_monitor.sdk.auto_collection.dependency_metrics.time") + def test_track_failed_dependency_rate_time_none(self, time_mock): + time_mock.time.return_value = 100 + metrics_collector = dependency_metrics.DependencyMetrics( + meter=self._meter, + labels=self._test_labels, + span_processor=self._span_processor, + ) + dependency_metrics.dependency_map["last_time"] = None + obs = Observer( + callback=metrics_collector._track_failure_rate, + name="test", + description="test", + unit="test", + value_type=float, + meter=self._meter, + ) + metrics_collector._track_failure_rate(obs) + self.assertEqual( + obs.aggregators[tuple(self._test_labels.items())].current, 0.0 + ) + + @mock.patch("azure_monitor.sdk.auto_collection.dependency_metrics.time") + def test_track_failed_dependency_rate_error(self, time_mock): + time_mock.time.return_value = 100 + metrics_collector = dependency_metrics.DependencyMetrics( + meter=self._meter, + labels=self._test_labels, + span_processor=self._span_processor, + ) + dependency_metrics.dependency_map["last_time"] = 100 + dependency_metrics.dependency_map["last_result"] = 5.0 + obs = Observer( + callback=metrics_collector._track_failure_rate, + name="test", + description="test", + unit="test", + value_type=float, + meter=self._meter, + ) + metrics_collector._track_failure_rate(obs) + self.assertEqual( + obs.aggregators[tuple(self._test_labels.items())].current, 5.0 + ) + + def test_track_dependency_duration(self): + metrics_collector = dependency_metrics.DependencyMetrics( + meter=self._meter, + labels=self._test_labels, + span_processor=self._span_processor, + ) + self._span_processor.dependency_duration = 100 + self._span_processor.dependency_count = 10 + dependency_metrics.dependency_map["last_count"] = 5 + obs = Observer( + callback=metrics_collector._track_dependency_duration, + name="test", + description="test", + unit="test", + value_type=int, + meter=self._meter, + ) + metrics_collector._track_dependency_duration(obs) + self.assertEqual( + obs.aggregators[tuple(self._test_labels.items())].current, 20 + ) + + def test_track_dependency_duration_error(self): + metrics_collector = dependency_metrics.DependencyMetrics( + meter=self._meter, + labels=self._test_labels, + span_processor=self._span_processor, + ) + self._span_processor.dependency_duration = 100 + self._span_processor.dependency_count = 10 + dependency_metrics.dependency_map["last_count"] = 10 + obs = Observer( + callback=metrics_collector._track_dependency_duration, + name="test", + description="test", + unit="test", + value_type=int, + meter=self._meter, + ) + metrics_collector._track_dependency_duration(obs) + self.assertEqual( + obs.aggregators[tuple(self._test_labels.items())].current, 0 + ) diff --git a/azure_monitor/tests/auto_collection/test_metrics_span_processor.py b/azure_monitor/tests/auto_collection/test_metrics_span_processor.py new file mode 100644 index 0000000..216ca34 --- /dev/null +++ b/azure_monitor/tests/auto_collection/test_metrics_span_processor.py @@ -0,0 +1,138 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import unittest + +from opentelemetry.sdk.trace import Span +from opentelemetry.trace import SpanContext, SpanKind +from opentelemetry.trace.status import Status, StatusCanonicalCode + +from azure_monitor.sdk.auto_collection.metrics_span_processor import ( + AzureMetricsSpanProcessor, +) + + +# pylint: disable=protected-access +class TestAutoCollection(unittest.TestCase): + def test_constructor(self): + """Test the constructor.""" + span_processor = AzureMetricsSpanProcessor() + self.assertEqual(span_processor.dependency_count, 0) + self.assertEqual(span_processor.dependency_duration, 0) + self.assertEqual(span_processor.failed_dependency_count, 0) + self.assertEqual(span_processor.request_count, 0) + self.assertEqual(span_processor.request_duration, 0) + self.assertEqual(span_processor.failed_request_count, 0) + + def test_ok_dependency(self): + """Test the functionality when Client Span is ended.""" + span_processor = AzureMetricsSpanProcessor() + test_span = Span( + name="test", + kind=SpanKind.CLIENT, + context=SpanContext( + trace_id=36873507687745823477771305566750195431, + span_id=12030755672171557338, + is_remote=False, + ), + ) + test_span._start_time = 5000000 + test_span._end_time = 15000000 + span_processor.on_end(test_span) + self.assertEqual(span_processor.request_count, 0) + self.assertEqual(span_processor.request_duration, 0) + self.assertEqual(span_processor.failed_request_count, 0) + self.assertEqual(span_processor.dependency_count, 1) + self.assertEqual(span_processor.dependency_duration, 10) + self.assertEqual(span_processor.failed_dependency_count, 0) + + def test_failed_dependency(self): + """Test the functionality when Client Span is ended.""" + span_processor = AzureMetricsSpanProcessor() + test_span = Span( + name="test", + kind=SpanKind.CLIENT, + context=SpanContext( + trace_id=36873507687745823477771305566750195431, + span_id=12030755672171557338, + is_remote=False, + ), + ) + test_span.set_status(Status(StatusCanonicalCode.INTERNAL, "test")) + test_span._start_time = 5000000 + test_span._end_time = 15000000 + span_processor.on_end(test_span) + self.assertEqual(span_processor.request_count, 0) + self.assertEqual(span_processor.request_duration, 0) + self.assertEqual(span_processor.failed_request_count, 0) + self.assertEqual(span_processor.dependency_count, 1) + self.assertEqual(span_processor.dependency_duration, 10) + self.assertEqual(span_processor.failed_dependency_count, 1) + + def test_ok_request(self): + """Test the functionality when Server Span is ended.""" + span_processor = AzureMetricsSpanProcessor() + test_span = Span( + name="test", + kind=SpanKind.SERVER, + context=SpanContext( + trace_id=36873507687745823477771305566750195431, + span_id=12030755672171557338, + is_remote=False, + ), + ) + test_span._start_time = 5000000 + test_span._end_time = 15000000 + span_processor.on_end(test_span) + self.assertEqual(span_processor.dependency_count, 0) + self.assertEqual(span_processor.dependency_duration, 0) + self.assertEqual(span_processor.failed_dependency_count, 0) + self.assertEqual(span_processor.request_count, 1) + self.assertEqual(span_processor.request_duration, 10) + self.assertEqual(span_processor.failed_request_count, 0) + + def test_failed_request(self): + """Test the functionality when Server Span is ended.""" + span_processor = AzureMetricsSpanProcessor() + test_span = Span( + name="test", + kind=SpanKind.SERVER, + context=SpanContext( + trace_id=36873507687745823477771305566750195431, + span_id=12030755672171557338, + is_remote=False, + ), + ) + test_span.set_status(Status(StatusCanonicalCode.INTERNAL, "test")) + test_span._start_time = 5000000 + test_span._end_time = 15000000 + span_processor.on_end(test_span) + self.assertEqual(span_processor.dependency_count, 0) + self.assertEqual(span_processor.dependency_duration, 0) + self.assertEqual(span_processor.failed_dependency_count, 0) + self.assertEqual(span_processor.request_count, 1) + self.assertEqual(span_processor.request_duration, 10) + self.assertEqual(span_processor.failed_request_count, 1) + + def test_document_collection(self): + """Test the document collection.""" + span_processor = AzureMetricsSpanProcessor() + span_processor.is_collecting_documents = True + test_span = Span( + name="test", + kind=SpanKind.SERVER, + context=SpanContext( + trace_id=36873507687745823477771305566750195431, + span_id=12030755672171557338, + is_remote=False, + ), + ) + test_span.set_status(Status(StatusCanonicalCode.INTERNAL, "test")) + test_span._start_time = 5000000 + test_span._end_time = 15000000 + span_processor.on_end(test_span) + document = span_processor.documents.pop() + self.assertIsNotNone(document) + self.assertEqual( + document.name, "Microsoft.ApplicationInsights.Request" + ) diff --git a/azure_monitor/tests/auto_collection/test_performance_metrics.py b/azure_monitor/tests/auto_collection/test_performance_metrics.py index ec81d2a..0474a22 100644 --- a/azure_monitor/tests/auto_collection/test_performance_metrics.py +++ b/azure_monitor/tests/auto_collection/test_performance_metrics.py @@ -9,6 +9,7 @@ from opentelemetry.sdk.metrics import MeterProvider, Observer from azure_monitor.sdk.auto_collection import PerformanceMetrics +from azure_monitor.sdk.auto_collection.utils import AutoCollectionType def throw(exc_type, *args, **kwargs): @@ -30,10 +31,12 @@ def setUpClass(cls): def tearDownClass(cls): metrics._METER_PROVIDER = None - def test_constructor(self): + def test_constructor_standard_metrics(self): mock_meter = mock.Mock() performance_metrics_collector = PerformanceMetrics( - meter=mock_meter, labels=self._test_labels + meter=mock_meter, + labels=self._test_labels, + collection_type=AutoCollectionType.STANDARD_METRICS, ) self.assertEqual(performance_metrics_collector._meter, mock_meter) self.assertEqual( @@ -70,9 +73,39 @@ def test_constructor(self): value_type=int, ) + def test_constructor_live_metrics(self): + mock_meter = mock.Mock() + performance_metrics_collector = PerformanceMetrics( + meter=mock_meter, + labels=self._test_labels, + collection_type=AutoCollectionType.LIVE_METRICS, + ) + self.assertEqual(performance_metrics_collector._meter, mock_meter) + self.assertEqual( + performance_metrics_collector._labels, self._test_labels + ) + self.assertEqual(mock_meter.register_observer.call_count, 2) + reg_obs_calls = mock_meter.register_observer.call_args_list + reg_obs_calls[0].assert_called_with( + callback=performance_metrics_collector._track_cpu, + name="\\Processor(_Total)\\% Processor Time", + description="Processor time as a percentage", + unit="percentage", + value_type=float, + ) + reg_obs_calls[1].assert_called_with( + callback=performance_metrics_collector._track_commited_memory, + name="\\Memory\\Committed Bytes", + description="Amount of commited memory in bytes", + unit="byte", + value_type=int, + ) + def test_track_cpu(self): performance_metrics_collector = PerformanceMetrics( - meter=self._meter, labels=self._test_labels + meter=self._meter, + labels=self._test_labels, + collection_type=AutoCollectionType.STANDARD_METRICS, ) with mock.patch("psutil.cpu_times_percent") as processor_mock: cpu = collections.namedtuple("cpu", "idle") @@ -94,7 +127,9 @@ def test_track_cpu(self): @mock.patch("psutil.virtual_memory") def test_track_memory(self, psutil_mock): performance_metrics_collector = PerformanceMetrics( - meter=self._meter, labels=self._test_labels + meter=self._meter, + labels=self._test_labels, + collection_type=AutoCollectionType.STANDARD_METRICS, ) memory = collections.namedtuple("memory", "available") vmem = memory(available=100) @@ -112,13 +147,38 @@ def test_track_memory(self, psutil_mock): obs.aggregators[tuple(self._test_labels.items())].current, 100 ) + @mock.patch("psutil.virtual_memory") + def test_track_commited_memory(self, psutil_mock): + performance_metrics_collector = PerformanceMetrics( + meter=self._meter, + labels=self._test_labels, + collection_type=AutoCollectionType.LIVE_METRICS, + ) + memory = collections.namedtuple("memory", ["available", "total"]) + vmem = memory(available=100, total=150) + psutil_mock.return_value = vmem + obs = Observer( + callback=performance_metrics_collector._track_commited_memory, + name="\\Memory\\Available Bytes", + description="Amount of available memory in bytes", + unit="byte", + value_type=int, + meter=self._meter, + ) + performance_metrics_collector._track_commited_memory(obs) + self.assertEqual( + obs.aggregators[tuple(self._test_labels.items())].current, 50 + ) + @mock.patch("azure_monitor.sdk.auto_collection.performance_metrics.psutil") def test_track_process_cpu(self, psutil_mock): with mock.patch( "azure_monitor.sdk.auto_collection.performance_metrics.PROCESS" ) as process_mock: performance_metrics_collector = PerformanceMetrics( - meter=self._meter, labels=self._test_labels + meter=self._meter, + labels=self._test_labels, + collection_type=AutoCollectionType.STANDARD_METRICS, ) process_mock.cpu_percent.return_value = 44.4 psutil_mock.cpu_count.return_value = 2 @@ -141,7 +201,9 @@ def test_track_process_cpu_exception(self, logger_mock): "azure_monitor.sdk.auto_collection.performance_metrics.psutil" ) as psutil_mock: performance_metrics_collector = PerformanceMetrics( - meter=self._meter, labels=self._test_labels + meter=self._meter, + labels=self._test_labels, + collection_type=AutoCollectionType.STANDARD_METRICS, ) psutil_mock.cpu_count.return_value = None obs = Observer( @@ -160,7 +222,9 @@ def test_track_process_memory(self): "azure_monitor.sdk.auto_collection.performance_metrics.PROCESS" ) as process_mock: performance_metrics_collector = PerformanceMetrics( - meter=self._meter, labels=self._test_labels + meter=self._meter, + labels=self._test_labels, + collection_type=AutoCollectionType.STANDARD_METRICS, ) memory = collections.namedtuple("memory", "rss") pmem = memory(rss=100) @@ -185,7 +249,9 @@ def test_track_process_memory_exception(self, logger_mock): throw(Exception), ): performance_metrics_collector = PerformanceMetrics( - meter=self._meter, labels=self._test_labels + meter=self._meter, + labels=self._test_labels, + collection_type=AutoCollectionType.STANDARD_METRICS, ) obs = Observer( callback=performance_metrics_collector._track_process_memory, diff --git a/azure_monitor/tests/auto_collection/test_request_metrics.py b/azure_monitor/tests/auto_collection/test_request_metrics.py index 62b5a88..2d0b121 100644 --- a/azure_monitor/tests/auto_collection/test_request_metrics.py +++ b/azure_monitor/tests/auto_collection/test_request_metrics.py @@ -2,17 +2,15 @@ # Licensed under the MIT License. import unittest -from http.server import HTTPServer from unittest import mock -import requests from opentelemetry import metrics from opentelemetry.sdk.metrics import MeterProvider, Observer from azure_monitor.sdk.auto_collection import request_metrics - -ORIGINAL_FUNCTION = requests.Session.request -ORIGINAL_CONS = HTTPServer.__init__ +from azure_monitor.sdk.auto_collection.metrics_span_processor import ( + AzureMetricsSpanProcessor, +) # pylint: disable=protected-access @@ -22,6 +20,7 @@ def setUpClass(cls): metrics.set_meter_provider(MeterProvider()) cls._meter = metrics.get_meter(__name__) cls._test_labels = {"environment": "staging"} + cls._span_processor = AzureMetricsSpanProcessor() @classmethod def tearDown(cls): @@ -29,22 +28,26 @@ def tearDown(cls): def setUp(self): request_metrics.requests_map.clear() - requests.Session.request = ORIGINAL_FUNCTION - request_metrics.ORIGINAL_CONSTRUCTOR = ORIGINAL_CONS def test_constructor(self): mock_meter = mock.Mock() request_metrics_collector = request_metrics.RequestMetrics( - meter=mock_meter, labels=self._test_labels + meter=mock_meter, + labels=self._test_labels, + span_processor=self._span_processor, ) self.assertEqual(request_metrics_collector._meter, mock_meter) self.assertEqual(request_metrics_collector._labels, self._test_labels) - - self.assertEqual(mock_meter.register_observer.call_count, 2) - + self.assertEqual(mock_meter.register_observer.call_count, 3) create_metric_calls = mock_meter.register_observer.call_args_list - create_metric_calls[0].assert_called_with( + callback=request_metrics_collector._track_request_failed_rate, + name="\\ApplicationInsights\\Requests Failed/Sec", + description="Incoming Requests Failed Rate", + unit="rps", + value_type=float, + ) + create_metric_calls[1].assert_called_with( callback=request_metrics_collector._track_request_duration, name="\\ASP.NET Applications(??APP_W3SVC_PROC??)\\Request Execution Time", description="Incoming Requests Average Execution Time", @@ -52,20 +55,22 @@ def test_constructor(self): value_type=int, ) - create_metric_calls[1].assert_called_with( + create_metric_calls[2].assert_called_with( callback=request_metrics_collector._track_request_rate, name="\\ASP.NET Applications(??APP_W3SVC_PROC??)\\Requests/Sec", - description="Incoming Requests Average Execution Rate", + description="Incoming Requests Rate", unit="rps", - value_type=int, + value_type=float, ) def test_track_request_duration(self): request_metrics_collector = request_metrics.RequestMetrics( - meter=self._meter, labels=self._test_labels + meter=self._meter, + labels=self._test_labels, + span_processor=self._span_processor, ) - request_metrics.requests_map["duration"] = 0.1 - request_metrics.requests_map["count"] = 10 + self._span_processor.request_duration = 100 + self._span_processor.request_count = 10 request_metrics.requests_map["last_count"] = 5 obs = Observer( callback=request_metrics_collector._track_request_duration, @@ -77,15 +82,17 @@ def test_track_request_duration(self): ) request_metrics_collector._track_request_duration(obs) self.assertEqual( - obs.aggregators[tuple(self._test_labels.items())].current, 20 + obs.aggregators[tuple(self._test_labels.items())].current, 20.0 ) def test_track_request_duration_error(self): request_metrics_collector = request_metrics.RequestMetrics( - meter=self._meter, labels=self._test_labels + meter=self._meter, + labels=self._test_labels, + span_processor=self._span_processor, ) - request_metrics.requests_map["duration"] = 0.1 - request_metrics.requests_map["count"] = 10 + self._span_processor.request_duration = 100 + self._span_processor.request_count = 10 request_metrics.requests_map["last_count"] = 10 obs = Observer( callback=request_metrics_collector._track_request_duration, @@ -97,35 +104,39 @@ def test_track_request_duration_error(self): ) request_metrics_collector._track_request_duration(obs) self.assertEqual( - obs.aggregators[tuple(self._test_labels.items())].current, 0 + obs.aggregators[tuple(self._test_labels.items())].current, 0.0 ) @mock.patch("azure_monitor.sdk.auto_collection.request_metrics.time") def test_track_request_rate(self, time_mock): request_metrics_collector = request_metrics.RequestMetrics( - meter=self._meter, labels=self._test_labels + meter=self._meter, + labels=self._test_labels, + span_processor=self._span_processor, ) time_mock.time.return_value = 100 request_metrics.requests_map["last_time"] = 98 - request_metrics.requests_map["count"] = 4 + self._span_processor.request_count = 4 obs = Observer( callback=request_metrics_collector._track_request_rate, name="\\ASP.NET Applications(??APP_W3SVC_PROC??)\\Requests/Sec", description="Incoming Requests Average Execution Rate", unit="rps", - value_type=int, + value_type=float, meter=self._meter, ) request_metrics_collector._track_request_rate(obs) self.assertEqual( - obs.aggregators[tuple(self._test_labels.items())].current, 2 + obs.aggregators[tuple(self._test_labels.items())].current, 2.0 ) @mock.patch("azure_monitor.sdk.auto_collection.request_metrics.time") def test_track_request_rate_time_none(self, time_mock): time_mock.time.return_value = 100 request_metrics_collector = request_metrics.RequestMetrics( - meter=self._meter, labels=self._test_labels + meter=self._meter, + labels=self._test_labels, + span_processor=self._span_processor, ) request_metrics.requests_map["last_time"] = None obs = Observer( @@ -133,92 +144,101 @@ def test_track_request_rate_time_none(self, time_mock): name="\\ASP.NET Applications(??APP_W3SVC_PROC??)\\Requests/Sec", description="Incoming Requests Average Execution Rate", unit="rps", - value_type=int, + value_type=float, meter=self._meter, ) request_metrics_collector._track_request_rate(obs) self.assertEqual( - obs.aggregators[tuple(self._test_labels.items())].current, 0 + obs.aggregators[tuple(self._test_labels.items())].current, 0.0 ) @mock.patch("azure_monitor.sdk.auto_collection.request_metrics.time") def test_track_request_rate_error(self, time_mock): request_metrics_collector = request_metrics.RequestMetrics( - meter=self._meter, labels=self._test_labels + meter=self._meter, + labels=self._test_labels, + span_processor=self._span_processor, ) time_mock.time.return_value = 100 - request_metrics.requests_map["last_rate"] = 5 + request_metrics.requests_map["last_rate"] = 5.0 request_metrics.requests_map["last_time"] = 100 obs = Observer( callback=request_metrics_collector._track_request_rate, name="\\ASP.NET Applications(??APP_W3SVC_PROC??)\\Requests/Sec", description="Incoming Requests Average Execution Rate", unit="rps", - value_type=int, + value_type=float, meter=self._meter, ) request_metrics_collector._track_request_rate(obs) self.assertEqual( - obs.aggregators[tuple(self._test_labels.items())].current, 5 - ) - - def test_request_patch(self): - map = request_metrics.requests_map # pylint: disable=redefined-builtin - func = mock.Mock() - new_func = request_metrics.request_patch(func) - new_func() - - self.assertEqual(map["count"], 1) - self.assertIsNotNone(map["duration"]) - self.assertEqual(len(func.call_args_list), 1) - - def test_server_patch(self): - request_metrics.ORIGINAL_CONSTRUCTOR = lambda x, y, z: None - with mock.patch( - "azure_monitor.sdk.auto_collection.request_metrics.request_patch" - ) as request_mock: - handler = mock.Mock() - handler.do_DELETE.return_value = None - handler.do_GET.return_value = None - handler.do_HEAD.return_value = None - handler.do_OPTIONS.return_value = None - handler.do_POST.return_value = None - handler.do_PUT.return_value = None - result = request_metrics.server_patch(None, None, handler) - handler.do_DELETE() - handler.do_GET() - handler.do_HEAD() - handler.do_OPTIONS() - handler.do_POST() - handler.do_PUT() - - self.assertEqual(result, None) - self.assertEqual(len(request_mock.call_args_list), 6) - - def test_server_patch_no_methods(self): - request_metrics.ORIGINAL_CONSTRUCTOR = lambda x, y, z: None - with mock.patch( - "azure_monitor.sdk.auto_collection.request_metrics.request_patch" - ) as request_mock: - handler = mock.Mock() - result = request_metrics.server_patch(None, None, handler) - handler.do_DELETE() - handler.do_GET() - handler.do_HEAD() - handler.do_OPTIONS() - handler.do_POST() - handler.do_PUT() - - self.assertEqual(result, None) - self.assertEqual(len(request_mock.call_args_list), 0) - - def test_server_patch_no_args(self): - request_metrics.ORIGINAL_CONSTRUCTOR = lambda x, y: None - req = request_metrics.server_patch(None, None) - - self.assertEqual(req, None) - - def test_server_patch_no_handler(self): - request_metrics.ORIGINAL_CONSTRUCTOR = lambda x, y, z: None - req = request_metrics.server_patch(None, None, None) - self.assertEqual(req, None) + obs.aggregators[tuple(self._test_labels.items())].current, 5.0 + ) + + @mock.patch("azure_monitor.sdk.auto_collection.request_metrics.time") + def test_track_request_failed_rate(self, time_mock): + request_metrics_collector = request_metrics.RequestMetrics( + meter=self._meter, + labels=self._test_labels, + span_processor=self._span_processor, + ) + time_mock.time.return_value = 100 + request_metrics.requests_map["last_time"] = 98 + self._span_processor.failed_request_count = 4 + obs = Observer( + callback=request_metrics_collector._track_request_failed_rate, + name="test", + description="test", + unit="test", + value_type=float, + meter=self._meter, + ) + request_metrics_collector._track_request_failed_rate(obs) + self.assertEqual( + obs.aggregators[tuple(self._test_labels.items())].current, 2.0 + ) + + @mock.patch("azure_monitor.sdk.auto_collection.request_metrics.time") + def test_track_request_failed_rate_time_none(self, time_mock): + time_mock.time.return_value = 100 + request_metrics_collector = request_metrics.RequestMetrics( + meter=self._meter, + labels=self._test_labels, + span_processor=self._span_processor, + ) + request_metrics.requests_map["last_time"] = None + obs = Observer( + callback=request_metrics_collector._track_request_failed_rate, + name="test", + description="test", + unit="test", + value_type=float, + meter=self._meter, + ) + request_metrics_collector._track_request_failed_rate(obs) + self.assertEqual( + obs.aggregators[tuple(self._test_labels.items())].current, 0.0 + ) + + @mock.patch("azure_monitor.sdk.auto_collection.request_metrics.time") + def test_track_request_failed_rate_error(self, time_mock): + request_metrics_collector = request_metrics.RequestMetrics( + meter=self._meter, + labels=self._test_labels, + span_processor=self._span_processor, + ) + time_mock.time.return_value = 100 + request_metrics.requests_map["last_rate"] = 5.0 + request_metrics.requests_map["last_time"] = 100 + obs = Observer( + callback=request_metrics_collector._track_request_failed_rate, + name="test", + description="test", + unit="test", + value_type=float, + meter=self._meter, + ) + request_metrics_collector._track_request_failed_rate(obs) + self.assertEqual( + obs.aggregators[tuple(self._test_labels.items())].current, 5.0 + )