Thanks to visit codestin.com
Credit goes to github.com

Skip to content
This repository was archived by the owner on Nov 16, 2023. It is now read-only.

Adding live metrics #96

Merged
merged 9 commits into from
Jun 15, 2020
Merged

Adding live metrics #96

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
02c3bd1
Adding live metrics
hectorhdzg Jun 3, 2020
e90e569
Merge branch 'master' into hectorh/liveMetr
hectorhdzg Jun 3, 2020
5173e55
Fixing lint issues
hectorhdzg Jun 3, 2020
52a293e
Merge branch 'hectorh/liveMetr' of https://github.com/hectorhdzg/open…
hectorhdzg Jun 3, 2020
db4914c
Using single classes for live/standard metrics to avoid duplication
hectorhdzg Jun 4, 2020
c18bf17
Adding tests
hectorhdzg Jun 5, 2020
fc4a642
Lint
hectorhdzg Jun 5, 2020
0c23ccc
Fixing issue with rates as int
hectorhdzg Jun 9, 2020
0a5a2e2
Fixing issue with documents parsing
hectorhdzg Jun 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 15 additions & 3 deletions azure_monitor/examples/metrics/auto_collector.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,21 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
from opentelemetry import metrics
from opentelemetry import metrics, trace
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export.controller import PushController
from opentelemetry.sdk.trace import TracerProvider

from azure_monitor import AzureMonitorMetricsExporter
from azure_monitor.sdk.auto_collection import AutoCollection
from azure_monitor.sdk.auto_collection import (
AutoCollection,
AzureMetricsSpanProcessor,
)

# Add Span Processor to get metrics about traces
span_processor = AzureMetricsSpanProcessor()
tracer_provider = TracerProvider()
tracer_provider.add_span_processor(span_processor)
trace.set_tracer_provider(tracer_provider)

metrics.set_meter_provider(MeterProvider())
meter = metrics.get_meter(__name__)
Expand All @@ -17,7 +27,9 @@
testing_label_set = {"environment": "testing"}

# Automatically collect standard metrics
auto_collection = AutoCollection(meter=meter, labels=testing_label_set)
auto_collection = AutoCollection(
meter=meter, labels=testing_label_set, span_processor=span_processor
)

# To configure a separate export interval specific for standard metrics
# meter_standard = metrics.get_meter(__name__ + "_standard")
Expand Down
107 changes: 107 additions & 0 deletions azure_monitor/src/azure_monitor/export/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,16 @@
import logging
import typing
from enum import Enum
from urllib.parse import urlparse

import requests
from opentelemetry.sdk.metrics.export import MetricsExportResult
from opentelemetry.sdk.trace.export import SpanExportResult
from opentelemetry.sdk.util import ns_to_iso_str
from opentelemetry.trace import Span, SpanKind
from opentelemetry.trace.status import StatusCanonicalCode

from azure_monitor import protocol, utils
from azure_monitor.options import ExporterOptions
from azure_monitor.protocol import Envelope
from azure_monitor.storage import LocalFileStorage
Expand Down Expand Up @@ -204,3 +209,105 @@ def get_metrics_export_result(result: ExportResult) -> MetricsExportResult:
):
return MetricsExportResult.FAILURE
return None


# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
def convert_span_to_envelope(span: Span) -> protocol.Envelope:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why move this here?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Need this conversion in LiveMetrics processing, could move to a more appropriate location like utility or something, any suggestion?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we leave this in trace and have LiveMetrics just import from trace module?

if not span:
return None
envelope = protocol.Envelope(
ikey="",
tags=dict(utils.azure_monitor_context),
time=ns_to_iso_str(span.start_time),
)
envelope.tags["ai.operation.id"] = "{:032x}".format(span.context.trace_id)
parent = span.parent
if isinstance(parent, Span):
parent = parent.context
if parent:
envelope.tags["ai.operation.parentId"] = "{:016x}".format(
parent.span_id
)
if span.kind in (SpanKind.CONSUMER, SpanKind.SERVER):
envelope.name = "Microsoft.ApplicationInsights.Request"
data = protocol.Request(
id="{:016x}".format(span.context.span_id),
duration=utils.ns_to_duration(span.end_time - span.start_time),
response_code=str(span.status.canonical_code.value),
success=span.status.canonical_code
== StatusCanonicalCode.OK, # Modify based off attributes or Status
properties={},
)
envelope.data = protocol.Data(base_data=data, base_type="RequestData")
if "http.method" in span.attributes:
data.name = span.attributes["http.method"]
if "http.route" in span.attributes:
data.name = data.name + " " + span.attributes["http.route"]
envelope.tags["ai.operation.name"] = data.name
data.properties["request.name"] = data.name
elif "http.path" in span.attributes:
data.properties["request.name"] = (
data.name + " " + span.attributes["http.path"]
)
if "http.url" in span.attributes:
data.url = span.attributes["http.url"]
data.properties["request.url"] = span.attributes["http.url"]
if "http.status_code" in span.attributes:
status_code = span.attributes["http.status_code"]
data.response_code = str(status_code)
data.success = 200 <= status_code < 400
else:
envelope.name = "Microsoft.ApplicationInsights.RemoteDependency"
data = protocol.RemoteDependency(
name=span.name,
id="{:016x}".format(span.context.span_id),
result_code=str(span.status.canonical_code.value),
duration=utils.ns_to_duration(span.end_time - span.start_time),
success=span.status.canonical_code
== StatusCanonicalCode.OK, # Modify based off attributes or Status
properties={},
)
envelope.data = protocol.Data(
base_data=data, base_type="RemoteDependencyData"
)
if span.kind in (SpanKind.CLIENT, SpanKind.PRODUCER):
if (
"component" in span.attributes
and span.attributes["component"] == "http"
):
data.type = "HTTP"
if "http.url" in span.attributes:
url = span.attributes["http.url"]
# data is the url
data.data = url
parse_url = urlparse(url)
# TODO: error handling, probably put scheme as well
# target matches authority (host:port)
data.target = parse_url.netloc
if "http.method" in span.attributes:
# name is METHOD/path
data.name = (
span.attributes["http.method"] + "/" + parse_url.path
)
if "http.status_code" in span.attributes:
status_code = span.attributes["http.status_code"]
data.result_code = str(status_code)
data.success = 200 <= status_code < 400
else: # SpanKind.INTERNAL
data.type = "InProc"
data.success = True
for key in span.attributes:
# This removes redundant data from ApplicationInsights
if key.startswith("http."):
continue
data.properties[key] = span.attributes[key]
if span.links:
links = []
for link in span.links:
operation_id = "{:032x}".format(link.context.trace_id)
span_id = "{:016x}".format(link.context.span_id)
links.append({"operation_Id": operation_id, "id": span_id})
data.properties["_MS.links"] = json.dumps(links)
# TODO: tracestate, tags
return envelope
103 changes: 3 additions & 100 deletions azure_monitor/src/azure_monitor/export/trace/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from azure_monitor.export import (
BaseExporter,
ExportResult,
convert_span_to_envelope,
get_trace_export_result,
)

Expand Down Expand Up @@ -52,104 +53,6 @@ def export(self, spans: Sequence[Span]) -> SpanExportResult:
def _span_to_envelope(self, span: Span) -> protocol.Envelope:
if not span:
return None
envelope = protocol.Envelope(
ikey=self.options.instrumentation_key,
tags=dict(utils.azure_monitor_context),
time=ns_to_iso_str(span.start_time),
)
envelope.tags["ai.operation.id"] = "{:032x}".format(
span.context.trace_id
)
parent = span.parent
if isinstance(parent, Span):
parent = parent.context
if parent:
envelope.tags["ai.operation.parentId"] = "{:016x}".format(
parent.span_id
)
if span.kind in (SpanKind.CONSUMER, SpanKind.SERVER):
envelope.name = "Microsoft.ApplicationInsights.Request"
data = protocol.Request(
id="{:016x}".format(span.context.span_id),
duration=utils.ns_to_duration(span.end_time - span.start_time),
response_code=str(span.status.canonical_code.value),
success=span.status.canonical_code
== StatusCanonicalCode.OK, # Modify based off attributes or Status
properties={},
)
envelope.data = protocol.Data(
base_data=data, base_type="RequestData"
)
if "http.method" in span.attributes:
data.name = span.attributes["http.method"]
if "http.route" in span.attributes:
data.name = data.name + " " + span.attributes["http.route"]
envelope.tags["ai.operation.name"] = data.name
data.properties["request.name"] = data.name
elif "http.path" in span.attributes:
data.properties["request.name"] = (
data.name + " " + span.attributes["http.path"]
)
if "http.url" in span.attributes:
data.url = span.attributes["http.url"]
data.properties["request.url"] = span.attributes["http.url"]
if "http.status_code" in span.attributes:
status_code = span.attributes["http.status_code"]
data.response_code = str(status_code)
data.success = 200 <= status_code < 400
else:
envelope.name = "Microsoft.ApplicationInsights.RemoteDependency"
data = protocol.RemoteDependency(
name=span.name,
id="{:016x}".format(span.context.span_id),
result_code=str(span.status.canonical_code.value),
duration=utils.ns_to_duration(span.end_time - span.start_time),
success=span.status.canonical_code
== StatusCanonicalCode.OK, # Modify based off attributes or Status
properties={},
)
envelope.data = protocol.Data(
base_data=data, base_type="RemoteDependencyData"
)
if span.kind in (SpanKind.CLIENT, SpanKind.PRODUCER):
if (
"component" in span.attributes
and span.attributes["component"] == "http"
):
data.type = "HTTP"
if "http.url" in span.attributes:
url = span.attributes["http.url"]
# data is the url
data.data = url
parse_url = urlparse(url)
# TODO: error handling, probably put scheme as well
# target matches authority (host:port)
data.target = parse_url.netloc
if "http.method" in span.attributes:
# name is METHOD/path
data.name = (
span.attributes["http.method"]
+ "/"
+ parse_url.path
)
if "http.status_code" in span.attributes:
status_code = span.attributes["http.status_code"]
data.result_code = str(status_code)
data.success = 200 <= status_code < 400
else: # SpanKind.INTERNAL
data.type = "InProc"
data.success = True
for key in span.attributes:
# This removes redundant data from ApplicationInsights
if key.startswith("http."):
continue
data.properties[key] = span.attributes[key]
if span.links:
links = []
for link in span.links:
operation_id = "{:032x}".format(link.context.trace_id)
span_id = "{:016x}".format(link.context.span_id)
links.append({"operation_Id": operation_id, "id": span_id})
data.properties["_MS.links"] = json.dumps(links)
# TODO: tracestate, tags
envelope = convert_span_to_envelope(span)
envelope.ikey = self.options.instrumentation_key
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How come we can't set the ikey in the convert function?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ikey is not available everywhere in the SDK, is currently specific to exporters, validation code is already in there, will need to centralize ikey management to be able to add the ikey in some shared function

return envelope
27 changes: 9 additions & 18 deletions azure_monitor/src/azure_monitor/protocol.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -563,19 +563,10 @@ def to_dict(self):
}


class LiveMetricDocumentProperty(BaseObject):

__slots__ = ("key", "value")

def __init__(self, key: str, value: str) -> None:
self.key = key
self.value = value


class LiveMetricDocument(BaseObject):

__slots__ = (
"__type",
"quickpulse_type",
"document_type",
"version",
"operation_id",
Expand All @@ -584,27 +575,25 @@ class LiveMetricDocument(BaseObject):

def __init__(
self,
__type: str = "",
quickpulse_type: str = "",
document_type: str = "",
version: str = "",
operation_id: str = "",
properties: typing.List[LiveMetricDocumentProperty] = None,
properties: typing.Dict[str, any] = None,
) -> None:
self.__type = __type
self.quickpulse_type = quickpulse_type
self.document_type = document_type
self.version = version
self.operation_id = operation_id
self.properties = properties

def to_dict(self):
return {
"__type": self.__type,
"__type": self.quickpulse_type,
"DocumentType": self.document_type,
"Version": self.version,
"OperationId": self.operation_id,
"Properties": self.properties.to_dict()
if self.properties
else None,
"Properties": self.properties,
}


Expand Down Expand Up @@ -675,7 +664,9 @@ def __init__(

def to_dict(self):
return {
"Documents": self.documents.to_dict() if self.documents else None,
"Documents": list(map(lambda x: x.to_dict(), self.documents))
if self.documents
else None,
"Instance": self.instance,
"InstrumentationKey": self.instrumentation_key,
"InvariantVersion": self.invariant_version,
Expand Down
22 changes: 18 additions & 4 deletions azure_monitor/src/azure_monitor/sdk/auto_collection/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,19 @@
from azure_monitor.sdk.auto_collection.dependency_metrics import (
DependencyMetrics,
)
from azure_monitor.sdk.auto_collection.metrics_span_processor import (
AzureMetricsSpanProcessor,
)
from azure_monitor.sdk.auto_collection.performance_metrics import (
PerformanceMetrics,
)
from azure_monitor.sdk.auto_collection.request_metrics import RequestMetrics
from azure_monitor.sdk.auto_collection.utils import AutoCollectionType

__all__ = [
"AutoCollection",
"AutoCollectionType",
"AzureMetricsSpanProcessor",
"DependencyMetrics",
"RequestMetrics",
"PerformanceMetrics",
Expand All @@ -30,7 +36,15 @@ class AutoCollection:
labels: Dictionary of labels
"""

def __init__(self, meter: Meter, labels: Dict[str, str]):
self._performance_metrics = PerformanceMetrics(meter, labels)
self._dependency_metrics = DependencyMetrics(meter, labels)
self._request_metrics = RequestMetrics(meter, labels)
def __init__(
self,
meter: Meter,
labels: Dict[str, str],
span_processor: AzureMetricsSpanProcessor,
):
col_type = AutoCollectionType.STANDARD_METRICS
self._performance_metrics = PerformanceMetrics(meter, labels, col_type)
self._dependency_metrics = DependencyMetrics(
meter, labels, span_processor
)
self._request_metrics = RequestMetrics(meter, labels, span_processor)
Loading