From 214bec8ca4471aa61d68e1f437d89cc86cdcb4a8 Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Wed, 4 Jan 2023 01:38:05 -0800 Subject: [PATCH 01/12] Initial implementation for supporting High Resolution Metrics --- aws_embedded_metrics/logger/metric.py | 3 +- .../logger/metrics_context.py | 7 +-- aws_embedded_metrics/logger/metrics_logger.py | 4 +- .../serializers/log_serializer.py | 5 +- aws_embedded_metrics/storageResolution.py | 16 +++++++ aws_embedded_metrics/validator.py | 14 +++++- tests/canary/agent/canary.py | 2 +- tests/logger/test_metrics_context.py | 47 +++++++++++++------ tests/logger/test_metrics_logger.py | 47 ++++++++++++++++++- tests/serializer/test_log_serializer.py | 23 +++++++++ 10 files changed, 144 insertions(+), 24 deletions(-) create mode 100644 aws_embedded_metrics/storageResolution.py diff --git a/aws_embedded_metrics/logger/metric.py b/aws_embedded_metrics/logger/metric.py index 7ef419f..0d40de3 100644 --- a/aws_embedded_metrics/logger/metric.py +++ b/aws_embedded_metrics/logger/metric.py @@ -13,9 +13,10 @@ class Metric(object): - def __init__(self, value: float, unit: str = None): + def __init__(self, value: float, unit: str = None, storageResolution: int = 60): self.values = [value] self.unit = unit or "None" + self.storageResolution = storageResolution def add_value(self, value: float) -> None: self.values.append(value) diff --git a/aws_embedded_metrics/logger/metrics_context.py b/aws_embedded_metrics/logger/metrics_context.py index bcb1cea..6a53034 100644 --- a/aws_embedded_metrics/logger/metrics_context.py +++ b/aws_embedded_metrics/logger/metrics_context.py @@ -39,8 +39,9 @@ def __init__( self.metrics: Dict[str, Metric] = {} self.should_use_default_dimensions = True self.meta: Dict[str, Any] = {"Timestamp": utils.now()} + self.metricNameAndResolutionMap: Dict[str, int] = {} - def put_metric(self, key: str, value: float, unit: str = None) -> None: + def put_metric(self, key: str, value: float, unit: str = None, storageResolution: int = 60) -> None: """ Adds a metric measurement to the context. Multiple calls using the same key will be stored as an @@ -49,13 +50,13 @@ def put_metric(self, key: str, value: float, unit: str = None) -> None: context.put_metric("Latency", 100, "Milliseconds") ``` """ - validate_metric(key, value, unit) + validate_metric(key, value, unit, storageResolution, self.metricNameAndResolutionMap) metric = self.metrics.get(key) if metric: # TODO: we should log a warning if the unit has been changed metric.add_value(value) else: - self.metrics[key] = Metric(value, unit) + self.metrics[key] = Metric(value, unit, storageResolution) def put_dimensions(self, dimension_set: Dict[str, str]) -> None: """ diff --git a/aws_embedded_metrics/logger/metrics_logger.py b/aws_embedded_metrics/logger/metrics_logger.py index e564e59..b26a553 100644 --- a/aws_embedded_metrics/logger/metrics_logger.py +++ b/aws_embedded_metrics/logger/metrics_logger.py @@ -78,8 +78,8 @@ def set_namespace(self, namespace: str) -> "MetricsLogger": self.context.namespace = namespace return self - def put_metric(self, key: str, value: float, unit: str = "None") -> "MetricsLogger": - self.context.put_metric(key, value, unit) + def put_metric(self, key: str, value: float, unit: str = "None", storageResolution: int = 60) -> "MetricsLogger": + self.context.put_metric(key, value, unit, storageResolution) return self def add_stack_trace(self, key: str, details: Any = None, exc_info: Tuple = None) -> "MetricsLogger": diff --git a/aws_embedded_metrics/serializers/log_serializer.py b/aws_embedded_metrics/serializers/log_serializer.py index 156e868..bd2ebf8 100644 --- a/aws_embedded_metrics/serializers/log_serializer.py +++ b/aws_embedded_metrics/serializers/log_serializer.py @@ -87,8 +87,11 @@ def create_body() -> Dict[str, Any]: if len(metric.values) > end_index: remaining_data = True + metricBody = {"Name": metric_name, "Unit": metric.unit} + if metric.storageResolution == 1: + metricBody["StorageResolution"] = metric.storageResolution if not config.disable_metric_extraction: - current_body["_aws"]["CloudWatchMetrics"][0]["Metrics"].append({"Name": metric_name, "Unit": metric.unit}) + current_body["_aws"]["CloudWatchMetrics"][0]["Metrics"].append(metricBody) num_metrics_in_current_body += 1 if (num_metrics_in_current_body == MAX_METRICS_PER_EVENT): diff --git a/aws_embedded_metrics/storageResolution.py b/aws_embedded_metrics/storageResolution.py new file mode 100644 index 0000000..73b999f --- /dev/null +++ b/aws_embedded_metrics/storageResolution.py @@ -0,0 +1,16 @@ +from enum import Enum, EnumMeta + + +class StorageResolutionMeta(EnumMeta): + def __contains__(self, item: object) -> bool: + try: + self(item) + except (ValueError, TypeError): + return False + else: + return True + + +class StorageResolution(Enum, metaclass=StorageResolutionMeta): + STANDARD = 60 + HIGH = 1 diff --git a/aws_embedded_metrics/validator.py b/aws_embedded_metrics/validator.py index b9e8395..9dd87dc 100644 --- a/aws_embedded_metrics/validator.py +++ b/aws_embedded_metrics/validator.py @@ -15,6 +15,7 @@ import re from typing import Dict, Optional from aws_embedded_metrics.unit import Unit +from aws_embedded_metrics.storageResolution import StorageResolution from aws_embedded_metrics.exceptions import DimensionSetExceededError, InvalidDimensionError, InvalidMetricError, InvalidNamespaceError import aws_embedded_metrics.constants as constants @@ -57,7 +58,7 @@ def validate_dimension_set(dimension_set: Dict[str, str]) -> None: raise InvalidDimensionError("Dimension name cannot start with ':'") -def validate_metric(name: str, value: float, unit: Optional[str]) -> None: +def validate_metric(name: str, value: float, unit: Optional[str], storageResolution: Optional[int], metricNameAndResolutionMap: Optional[dict]) -> None: # noqa: E501 """ Validates a metric @@ -65,6 +66,7 @@ def validate_metric(name: str, value: float, unit: Optional[str]) -> None: name (str): The name of the metric value (float): The value of the metric unit (Optional[str]): The unit of the metric + storageResolution (Optional[int]): The storage resolution of metric Raises: InvalidMetricError: If the metric is invalid @@ -81,6 +83,16 @@ def validate_metric(name: str, value: float, unit: Optional[str]) -> None: if unit is not None and unit not in Unit: raise InvalidMetricError(f"Metric unit is not valid: {unit}") + if storageResolution is None or storageResolution not in StorageResolution: + raise InvalidMetricError(f"Metric storage Resolution is not valid: {storageResolution}") + + if metricNameAndResolutionMap and name in metricNameAndResolutionMap: + if metricNameAndResolutionMap.get(name) is not storageResolution: + raise InvalidMetricError( + f"Resolution for metrics ${name} is already set. A single log event cannot have a metric with two different resolutions.") + else: + metricNameAndResolutionMap[name] = storageResolution + def validate_namespace(namespace: str) -> None: """ diff --git a/tests/canary/agent/canary.py b/tests/canary/agent/canary.py index 704e6ee..4a28321 100644 --- a/tests/canary/agent/canary.py +++ b/tests/canary/agent/canary.py @@ -26,7 +26,7 @@ async def app(init, last_run_duration, metrics): metrics.set_dimensions({"Runtime": 'Python', "Platform": 'ECS', "Agent": 'CloudWatchAgent', "Version": version}) metrics.put_metric('Invoke', 1, "Count") metrics.put_metric('Duration', last_run_duration, 'Seconds') - metrics.put_metric('Memory.RSS', process.memory_info().rss, 'Bytes') + metrics.put_metric('Memory.RSS', process.memory_info().rss, 'Bytes', 1) async def main(): diff --git a/tests/logger/test_metrics_context.py b/tests/logger/test_metrics_context.py index f20ed43..79f6f6d 100644 --- a/tests/logger/test_metrics_context.py +++ b/tests/logger/test_metrics_context.py @@ -3,6 +3,7 @@ import random from aws_embedded_metrics import constants from aws_embedded_metrics.unit import Unit +from aws_embedded_metrics.storageResolution import StorageResolution from aws_embedded_metrics import config from aws_embedded_metrics.logger.metrics_context import MetricsContext from aws_embedded_metrics.constants import DEFAULT_NAMESPACE @@ -263,14 +264,16 @@ def test_put_metric_adds_metrics(): metric_key = fake.word() metric_value = fake.random.random() metric_unit = random.choice(list(Unit)).value + metric_storageResolution = random.choice(list(StorageResolution)).value # act - context.put_metric(metric_key, metric_value, metric_unit) + context.put_metric(metric_key, metric_value, metric_unit, metric_storageResolution) # assert metric = context.metrics[metric_key] assert metric.unit == metric_unit assert metric.values == [metric_value] + assert metric.storageResolution == metric_storageResolution def test_put_metric_uses_none_unit_if_not_provided(): @@ -287,26 +290,42 @@ def test_put_metric_uses_none_unit_if_not_provided(): assert metric.unit == "None" +def test_put_metric_uses_60_storage_resolution_if_not_provided(): + # arrange + context = MetricsContext() + metric_key = fake.word() + metric_value = fake.random.random() + + # act + context.put_metric(metric_key, metric_value) + + # assert + metric = context.metrics[metric_key] + assert metric.storageResolution == 60 + + @pytest.mark.parametrize( - "name, value, unit", + "name, value, unit, storageResolution", [ - ("", 1, "None"), - (" ", 1, "Seconds"), - ("a" * (constants.MAX_METRIC_NAME_LENGTH + 1), 1, "None"), - ("metric", float("inf"), "Count"), - ("metric", float("-inf"), "Count"), - ("metric", float("nan"), "Count"), - ("metric", math.inf, "Seconds"), - ("metric", -math.inf, "Seconds"), - ("metric", math.nan, "Seconds"), - ("metric", 1, "Kilometers/Fahrenheit") + ("", 1, "None", 60), + (" ", 1, "Seconds", 60), + ("a" * (constants.MAX_METRIC_NAME_LENGTH + 1), 1, "None", 60), + ("metric", float("inf"), "Count", 60), + ("metric", float("-inf"), "Count", 60), + ("metric", float("nan"), "Count", 60), + ("metric", math.inf, "Seconds", 60), + ("metric", -math.inf, "Seconds", 60), + ("metric", math.nan, "Seconds", 60), + ("metric", 1, "Kilometers/Fahrenheit", 60), + ("metric", 1, "Seconds", 2), + ("metric", 1, "Seconds", None) ] ) -def test_put_invalid_metric_raises_exception(name, value, unit): +def test_put_invalid_metric_raises_exception(name, value, unit, storageResolution): context = MetricsContext() with pytest.raises(InvalidMetricError): - context.put_metric(name, value, unit) + context.put_metric(name, value, unit, storageResolution) def test_create_copy_with_context_creates_new_instance(): diff --git a/tests/logger/test_metrics_logger.py b/tests/logger/test_metrics_logger.py index dbcd088..35e3b5f 100644 --- a/tests/logger/test_metrics_logger.py +++ b/tests/logger/test_metrics_logger.py @@ -2,7 +2,7 @@ from aws_embedded_metrics.logger import metrics_logger from aws_embedded_metrics.sinks import Sink from aws_embedded_metrics.environment import Environment -from aws_embedded_metrics.exceptions import InvalidNamespaceError +from aws_embedded_metrics.exceptions import InvalidNamespaceError, InvalidMetricError import aws_embedded_metrics.constants as constants import pytest from faker import Faker @@ -53,6 +53,51 @@ async def test_can_put_metric(mocker): assert context.metrics[expected_key].unit == "None" +@pytest.mark.asyncio +async def test_can_put_metric_with_different_storage_resolution_different_flush(mocker): + # arrange + expected_key = fake.word() + expected_value = fake.random.randrange(100) + metric_storageResolution = 1 + + logger, sink, env = get_logger_and_sink(mocker) + + # act + logger.put_metric(expected_key, expected_value, None, metric_storageResolution) + await logger.flush() + + # assert + context = sink.accept.call_args[0][0] + assert context.metrics[expected_key].values == [expected_value] + assert context.metrics[expected_key].unit == "None" + assert context.metrics[expected_key].storageResolution == metric_storageResolution + + expected_key = fake.word() + expected_value = fake.random.randrange(100) + logger.put_metric(expected_key, expected_value, None) + await logger.flush() + context = sink.accept.call_args[0][0] + assert context.metrics[expected_key].values == [expected_value] + assert context.metrics[expected_key].unit == "None" + assert context.metrics[expected_key].storageResolution == 60 + + +@pytest.mark.asyncio +async def test_cannot_put_metric_with_different_storage_resolution_same_flush(mocker): + # arrange + expected_key = fake.word() + expected_value = fake.random.randrange(100) + metric_storageResolution = 1 + + logger, sink, env = get_logger_and_sink(mocker) + + # act + with pytest.raises(InvalidMetricError): + logger.put_metric(expected_key, expected_value, None, metric_storageResolution) + logger.put_metric(expected_key, expected_value, None, 60) + await logger.flush() + + @pytest.mark.asyncio async def test_can_add_stack_trace(mocker): # arrange diff --git a/tests/serializer/test_log_serializer.py b/tests/serializer/test_log_serializer.py index 3114b34..9abad2c 100644 --- a/tests/serializer/test_log_serializer.py +++ b/tests/serializer/test_log_serializer.py @@ -92,6 +92,29 @@ def test_serialize_metrics(): assert_json_equality(result_json, expected) +def test_serialize_metrics_with_storageResolution(): + # arrange + expected_key = fake.word() + expected_value = fake.random.randrange(0, 100) + + expected_metric_definition = {"Name": expected_key, "Unit": "None", "StorageResolution": 1} + + expected = {**get_empty_payload()} + expected[expected_key] = expected_value + expected["_aws"]["CloudWatchMetrics"][0]["Metrics"].append( + expected_metric_definition + ) + + context = get_context() + context.put_metric(expected_key, expected_value, "None", 1) + + # act + result_json = serializer.serialize(context)[0] + + # assert + assert_json_equality(result_json, expected) + + def test_serialize_more_than_100_metrics(): # arrange expected_value = fake.random.randrange(0, 100) From 6433a93a3db0a5cbc709f93e419293a92ebe4195 Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Wed, 4 Jan 2023 22:45:28 -0800 Subject: [PATCH 02/12] Minor fix --- aws_embedded_metrics/serializers/log_serializer.py | 2 +- aws_embedded_metrics/validator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_embedded_metrics/serializers/log_serializer.py b/aws_embedded_metrics/serializers/log_serializer.py index bd2ebf8..d008d33 100644 --- a/aws_embedded_metrics/serializers/log_serializer.py +++ b/aws_embedded_metrics/serializers/log_serializer.py @@ -89,7 +89,7 @@ def create_body() -> Dict[str, Any]: metricBody = {"Name": metric_name, "Unit": metric.unit} if metric.storageResolution == 1: - metricBody["StorageResolution"] = metric.storageResolution + metricBody["StorageResolution"] = metric.storageResolution # type: ignore if not config.disable_metric_extraction: current_body["_aws"]["CloudWatchMetrics"][0]["Metrics"].append(metricBody) num_metrics_in_current_body += 1 diff --git a/aws_embedded_metrics/validator.py b/aws_embedded_metrics/validator.py index 9dd87dc..1e04fca 100644 --- a/aws_embedded_metrics/validator.py +++ b/aws_embedded_metrics/validator.py @@ -58,7 +58,7 @@ def validate_dimension_set(dimension_set: Dict[str, str]) -> None: raise InvalidDimensionError("Dimension name cannot start with ':'") -def validate_metric(name: str, value: float, unit: Optional[str], storageResolution: Optional[int], metricNameAndResolutionMap: Optional[dict]) -> None: # noqa: E501 +def validate_metric(name: str, value: float, unit: Optional[str], storageResolution: Optional[int], metricNameAndResolutionMap: dict) -> None: # noqa: E501 """ Validates a metric From 6b1fa10fc4618045e9b4015998d28aeb727ee6de Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Tue, 10 Jan 2023 07:48:05 -0800 Subject: [PATCH 03/12] Updated README.md and examples folder --- README.md | 8 ++++---- examples/ec2/app.py | 1 + examples/lambda/function.py | 1 + 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3fa4368..5bf1e5e 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ from aws_embedded_metrics import metric_scope @metric_scope def my_handler(metrics): metrics.put_dimensions({"Foo": "Bar"}) - metrics.put_metric("ProcessingLatency", 100, "Milliseconds") + metrics.put_metric("ProcessingLatency", 100, "Milliseconds", 60) metrics.set_property("AccountId", "123456789012") metrics.set_property("RequestId", "422b1569-16f6-4a03") metrics.set_property("DeviceId", "61270781-c6ac-46f1") @@ -53,9 +53,9 @@ def my_handler(metrics): The `MetricsLogger` is the interface you will use to publish embedded metrics. -- **put_metric**(key: str, value: float, unit: str = "None") -> MetricsLogger +- **put_metric**(key: str, value: float, unit: str = "None", storageResolution: int = 60) -> MetricsLogger -Adds a new metric to the current logger context. Multiple metrics using the same key will be appended to an array of values. The Embedded Metric Format supports a maximum of 100 values per key. If more metric values are added than are supported by the format, the logger will be flushed to allow for new metric values to be captured. +Adds a new metric to the current logger context. Multiple metrics using the same key will be appended to an array of values. Multiple metrics cannot have same key and different storage resolution. The Embedded Metric Format supports a maximum of 100 values per key. If more metric values are added than are supported by the format, the logger will be flushed to allow for new metric values to be captured. Requirements: @@ -67,7 +67,7 @@ Requirements: Examples: ```py -put_metric("Latency", 200, "Milliseconds") +put_metric("Latency", 200, "Milliseconds", 60) ``` - **set_property**(key: str, value: Any) -> MetricsLogger diff --git a/examples/ec2/app.py b/examples/ec2/app.py index 13e9b6c..ac65947 100644 --- a/examples/ec2/app.py +++ b/examples/ec2/app.py @@ -10,6 +10,7 @@ def my_handler(metrics): metrics.put_dimensions({"Foo": "Bar"}) metrics.put_metric("ProcessingLatency", 100, "Milliseconds") + metrics.put_metric("CPU Utilization", 87, "Percent", 1) metrics.set_property("AccountId", "123456789012") metrics.set_property("RequestId", "422b1569-16f6-4a03-b8f0-fe3fd9b100f8") metrics.set_property("DeviceId", "61270781-c6ac-46f1-baf7-22c808af8162") diff --git a/examples/lambda/function.py b/examples/lambda/function.py index cd63b4d..37a4355 100644 --- a/examples/lambda/function.py +++ b/examples/lambda/function.py @@ -5,6 +5,7 @@ def my_handler(event, context, metrics): metrics.put_dimensions({"Foo": "Bar"}) metrics.put_metric("ProcessingLatency", 100, "Milliseconds") + metrics.put_metric("CPU Utilization", 87, "Percent", 1) metrics.set_property("AccountId", "123456789012") metrics.set_property("RequestId", "422b1569-16f6-4a03-b8f0-fe3fd9b100f8") metrics.set_property("DeviceId", "61270781-c6ac-46f1-baf7-22c808af8162") From 83e6e40a208df30f0f399ec569361b969af428e8 Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Wed, 11 Jan 2023 21:50:00 -0800 Subject: [PATCH 04/12] Move map updating outside validator and minor fix --- aws_embedded_metrics/logger/metrics_context.py | 2 ++ aws_embedded_metrics/validator.py | 3 +-- tests/logger/test_metrics_logger.py | 3 +-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/aws_embedded_metrics/logger/metrics_context.py b/aws_embedded_metrics/logger/metrics_context.py index 6a53034..2a9c6a2 100644 --- a/aws_embedded_metrics/logger/metrics_context.py +++ b/aws_embedded_metrics/logger/metrics_context.py @@ -57,6 +57,8 @@ def put_metric(self, key: str, value: float, unit: str = None, storageResolution metric.add_value(value) else: self.metrics[key] = Metric(value, unit, storageResolution) + if key not in self.metricNameAndResolutionMap: + self.metricNameAndResolutionMap[key] = storageResolution def put_dimensions(self, dimension_set: Dict[str, str]) -> None: """ diff --git a/aws_embedded_metrics/validator.py b/aws_embedded_metrics/validator.py index 1e04fca..bc5156b 100644 --- a/aws_embedded_metrics/validator.py +++ b/aws_embedded_metrics/validator.py @@ -67,6 +67,7 @@ def validate_metric(name: str, value: float, unit: Optional[str], storageResolut value (float): The value of the metric unit (Optional[str]): The unit of the metric storageResolution (Optional[int]): The storage resolution of metric + metricNameAndResolutionMap (dict): The map used for validating metric Raises: InvalidMetricError: If the metric is invalid @@ -90,8 +91,6 @@ def validate_metric(name: str, value: float, unit: Optional[str], storageResolut if metricNameAndResolutionMap.get(name) is not storageResolution: raise InvalidMetricError( f"Resolution for metrics ${name} is already set. A single log event cannot have a metric with two different resolutions.") - else: - metricNameAndResolutionMap[name] = storageResolution def validate_namespace(namespace: str) -> None: diff --git a/tests/logger/test_metrics_logger.py b/tests/logger/test_metrics_logger.py index 35e3b5f..02dbeab 100644 --- a/tests/logger/test_metrics_logger.py +++ b/tests/logger/test_metrics_logger.py @@ -87,13 +87,12 @@ async def test_cannot_put_metric_with_different_storage_resolution_same_flush(mo # arrange expected_key = fake.word() expected_value = fake.random.randrange(100) - metric_storageResolution = 1 logger, sink, env = get_logger_and_sink(mocker) # act + logger.put_metric(expected_key, expected_value, None, 1) with pytest.raises(InvalidMetricError): - logger.put_metric(expected_key, expected_value, None, metric_storageResolution) logger.put_metric(expected_key, expected_value, None, 60) await logger.flush() From 84c472021cdd4cf926a00ebb7573062c48aa28db Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Wed, 11 Jan 2023 23:15:57 -0800 Subject: [PATCH 05/12] Pass storageResolution as enum in putMetric --- aws_embedded_metrics/logger/metric.py | 5 ++-- .../logger/metrics_context.py | 5 ++-- aws_embedded_metrics/logger/metrics_logger.py | 5 +++- .../serializers/log_serializer.py | 5 ++-- aws_embedded_metrics/validator.py | 4 +-- examples/ec2/app.py | 3 +- examples/lambda/function.py | 3 +- tests/canary/agent/canary.py | 3 +- tests/integ/agent/test_end_to_end.py | 5 ++-- tests/logger/test_metrics_context.py | 24 ++++++++-------- tests/logger/test_metrics_logger.py | 9 +++--- tests/serializer/test_log_serializer.py | 28 +++++++++++++++++-- 12 files changed, 67 insertions(+), 32 deletions(-) diff --git a/aws_embedded_metrics/logger/metric.py b/aws_embedded_metrics/logger/metric.py index 0d40de3..a8adb15 100644 --- a/aws_embedded_metrics/logger/metric.py +++ b/aws_embedded_metrics/logger/metric.py @@ -10,13 +10,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from aws_embedded_metrics.storageResolution import StorageResolution class Metric(object): - def __init__(self, value: float, unit: str = None, storageResolution: int = 60): + def __init__(self, value: float, unit: str = None, storageResolution: StorageResolution = StorageResolution.STANDARD): self.values = [value] self.unit = unit or "None" - self.storageResolution = storageResolution + self.storageResolution = storageResolution or StorageResolution.STANDARD def add_value(self, value: float) -> None: self.values.append(value) diff --git a/aws_embedded_metrics/logger/metrics_context.py b/aws_embedded_metrics/logger/metrics_context.py index 2a9c6a2..6040a2d 100644 --- a/aws_embedded_metrics/logger/metrics_context.py +++ b/aws_embedded_metrics/logger/metrics_context.py @@ -16,6 +16,7 @@ from aws_embedded_metrics.config import get_config from aws_embedded_metrics.logger.metric import Metric from aws_embedded_metrics.validator import validate_dimension_set, validate_metric +from aws_embedded_metrics.storageResolution import StorageResolution from typing import List, Dict, Any, Set @@ -39,9 +40,9 @@ def __init__( self.metrics: Dict[str, Metric] = {} self.should_use_default_dimensions = True self.meta: Dict[str, Any] = {"Timestamp": utils.now()} - self.metricNameAndResolutionMap: Dict[str, int] = {} + self.metricNameAndResolutionMap: Dict[str, StorageResolution] = {} - def put_metric(self, key: str, value: float, unit: str = None, storageResolution: int = 60) -> None: + def put_metric(self, key: str, value: float, unit: str = None, storageResolution: StorageResolution = StorageResolution.STANDARD) -> None: """ Adds a metric measurement to the context. Multiple calls using the same key will be stored as an diff --git a/aws_embedded_metrics/logger/metrics_logger.py b/aws_embedded_metrics/logger/metrics_logger.py index b26a553..c89972f 100644 --- a/aws_embedded_metrics/logger/metrics_logger.py +++ b/aws_embedded_metrics/logger/metrics_logger.py @@ -15,6 +15,7 @@ from aws_embedded_metrics.logger.metrics_context import MetricsContext from aws_embedded_metrics.validator import validate_namespace from aws_embedded_metrics.config import get_config +from aws_embedded_metrics.storageResolution import StorageResolution from typing import Any, Awaitable, Callable, Dict, Tuple import sys import traceback @@ -78,7 +79,9 @@ def set_namespace(self, namespace: str) -> "MetricsLogger": self.context.namespace = namespace return self - def put_metric(self, key: str, value: float, unit: str = "None", storageResolution: int = 60) -> "MetricsLogger": + def put_metric( + self, key: str, value: float, unit: str = "None", storageResolution: StorageResolution = StorageResolution.STANDARD + ) -> "MetricsLogger": self.context.put_metric(key, value, unit, storageResolution) return self diff --git a/aws_embedded_metrics/serializers/log_serializer.py b/aws_embedded_metrics/serializers/log_serializer.py index d008d33..e63e4b8 100644 --- a/aws_embedded_metrics/serializers/log_serializer.py +++ b/aws_embedded_metrics/serializers/log_serializer.py @@ -18,6 +18,7 @@ MAX_DIMENSION_SET_SIZE, MAX_METRICS_PER_EVENT, MAX_DATAPOINTS_PER_METRIC ) from aws_embedded_metrics.exceptions import DimensionSetExceededError +from aws_embedded_metrics.storageResolution import StorageResolution import json from typing import Any, Dict, List @@ -88,8 +89,8 @@ def create_body() -> Dict[str, Any]: remaining_data = True metricBody = {"Name": metric_name, "Unit": metric.unit} - if metric.storageResolution == 1: - metricBody["StorageResolution"] = metric.storageResolution # type: ignore + if metric.storageResolution == StorageResolution.HIGH: + metricBody["StorageResolution"] = metric.storageResolution.value # type: ignore if not config.disable_metric_extraction: current_body["_aws"]["CloudWatchMetrics"][0]["Metrics"].append(metricBody) num_metrics_in_current_body += 1 diff --git a/aws_embedded_metrics/validator.py b/aws_embedded_metrics/validator.py index bc5156b..abd776b 100644 --- a/aws_embedded_metrics/validator.py +++ b/aws_embedded_metrics/validator.py @@ -58,7 +58,7 @@ def validate_dimension_set(dimension_set: Dict[str, str]) -> None: raise InvalidDimensionError("Dimension name cannot start with ':'") -def validate_metric(name: str, value: float, unit: Optional[str], storageResolution: Optional[int], metricNameAndResolutionMap: dict) -> None: # noqa: E501 +def validate_metric(name: str, value: float, unit: Optional[str], storageResolution: StorageResolution, metricNameAndResolutionMap: dict) -> None: # noqa: E501 """ Validates a metric @@ -90,7 +90,7 @@ def validate_metric(name: str, value: float, unit: Optional[str], storageResolut if metricNameAndResolutionMap and name in metricNameAndResolutionMap: if metricNameAndResolutionMap.get(name) is not storageResolution: raise InvalidMetricError( - f"Resolution for metrics ${name} is already set. A single log event cannot have a metric with two different resolutions.") + "Resolution for metrics ${name} is already set. A single log event cannot have a metric with two different resolutions.") def validate_namespace(namespace: str) -> None: diff --git a/examples/ec2/app.py b/examples/ec2/app.py index ac65947..46d38ee 100644 --- a/examples/ec2/app.py +++ b/examples/ec2/app.py @@ -1,4 +1,5 @@ from aws_embedded_metrics import metric_scope +from aws_embedded_metrics.storageResolution import StorageResolution import logging @@ -10,7 +11,7 @@ def my_handler(metrics): metrics.put_dimensions({"Foo": "Bar"}) metrics.put_metric("ProcessingLatency", 100, "Milliseconds") - metrics.put_metric("CPU Utilization", 87, "Percent", 1) + metrics.put_metric("CPU Utilization", 87, "Percent", StorageResolution.HIGH) metrics.set_property("AccountId", "123456789012") metrics.set_property("RequestId", "422b1569-16f6-4a03-b8f0-fe3fd9b100f8") metrics.set_property("DeviceId", "61270781-c6ac-46f1-baf7-22c808af8162") diff --git a/examples/lambda/function.py b/examples/lambda/function.py index 37a4355..a1b6657 100644 --- a/examples/lambda/function.py +++ b/examples/lambda/function.py @@ -1,11 +1,12 @@ from aws_embedded_metrics import metric_scope +from aws_embedded_metrics.storageResolution import StorageResolution @metric_scope def my_handler(event, context, metrics): metrics.put_dimensions({"Foo": "Bar"}) metrics.put_metric("ProcessingLatency", 100, "Milliseconds") - metrics.put_metric("CPU Utilization", 87, "Percent", 1) + metrics.put_metric("CPU Utilization", 87, "Percent", StorageResolution.HIGH) metrics.set_property("AccountId", "123456789012") metrics.set_property("RequestId", "422b1569-16f6-4a03-b8f0-fe3fd9b100f8") metrics.set_property("DeviceId", "61270781-c6ac-46f1-baf7-22c808af8162") diff --git a/tests/canary/agent/canary.py b/tests/canary/agent/canary.py index 4a28321..7e9113e 100644 --- a/tests/canary/agent/canary.py +++ b/tests/canary/agent/canary.py @@ -2,6 +2,7 @@ import aws_embedded_metrics from aws_embedded_metrics import metric_scope from aws_embedded_metrics.config import get_config +from aws_embedded_metrics.storageResolution import StorageResolution from getversion import get_module_version import os import psutil @@ -26,7 +27,7 @@ async def app(init, last_run_duration, metrics): metrics.set_dimensions({"Runtime": 'Python', "Platform": 'ECS', "Agent": 'CloudWatchAgent', "Version": version}) metrics.put_metric('Invoke', 1, "Count") metrics.put_metric('Duration', last_run_duration, 'Seconds') - metrics.put_metric('Memory.RSS', process.memory_info().rss, 'Bytes', 1) + metrics.put_metric('Memory.RSS', process.memory_info().rss, 'Bytes', StorageResolution.HIGH) async def main(): diff --git a/tests/integ/agent/test_end_to_end.py b/tests/integ/agent/test_end_to_end.py index 9bb363e..a47fb0f 100644 --- a/tests/integ/agent/test_end_to_end.py +++ b/tests/integ/agent/test_end_to_end.py @@ -1,5 +1,6 @@ from aws_embedded_metrics.config import get_config from aws_embedded_metrics import metric_scope +from aws_embedded_metrics.storageResolution import StorageResolution import pytest import boto3 import logging @@ -40,7 +41,7 @@ async def test_end_to_end_tcp_multiple_flushes(): @metric_scope async def do_work(metrics): metrics.put_dimensions({"Operation": "Agent"}) - metrics.put_metric(metric_name, 100, "Milliseconds") + metrics.put_metric(metric_name, 100, "Milliseconds", StorageResolution.HIGH) metrics.set_property("RequestId", "422b1569-16f6-4a03-b8f0-fe3fd9b100f8") # act @@ -70,7 +71,7 @@ async def test_end_to_end_udp(): @metric_scope async def do_work(metrics): metrics.put_dimensions({"Operation": "Agent"}) - metrics.put_metric(metric_name, 100, "Milliseconds") + metrics.put_metric(metric_name, 100, "Milliseconds", StorageResolution.HIGH) metrics.set_property("RequestId", "422b1569-16f6-4a03-b8f0-fe3fd9b100f8") # act diff --git a/tests/logger/test_metrics_context.py b/tests/logger/test_metrics_context.py index 79f6f6d..7dca3ef 100644 --- a/tests/logger/test_metrics_context.py +++ b/tests/logger/test_metrics_context.py @@ -290,7 +290,7 @@ def test_put_metric_uses_none_unit_if_not_provided(): assert metric.unit == "None" -def test_put_metric_uses_60_storage_resolution_if_not_provided(): +def test_put_metric_uses_standard_storage_resolution_if_not_provided(): # arrange context = MetricsContext() metric_key = fake.word() @@ -301,22 +301,22 @@ def test_put_metric_uses_60_storage_resolution_if_not_provided(): # assert metric = context.metrics[metric_key] - assert metric.storageResolution == 60 + assert metric.storageResolution == StorageResolution.STANDARD @pytest.mark.parametrize( "name, value, unit, storageResolution", [ - ("", 1, "None", 60), - (" ", 1, "Seconds", 60), - ("a" * (constants.MAX_METRIC_NAME_LENGTH + 1), 1, "None", 60), - ("metric", float("inf"), "Count", 60), - ("metric", float("-inf"), "Count", 60), - ("metric", float("nan"), "Count", 60), - ("metric", math.inf, "Seconds", 60), - ("metric", -math.inf, "Seconds", 60), - ("metric", math.nan, "Seconds", 60), - ("metric", 1, "Kilometers/Fahrenheit", 60), + ("", 1, "None", StorageResolution.STANDARD), + (" ", 1, "Seconds", StorageResolution.STANDARD), + ("a" * (constants.MAX_METRIC_NAME_LENGTH + 1), 1, "None", StorageResolution.STANDARD), + ("metric", float("inf"), "Count", StorageResolution.STANDARD), + ("metric", float("-inf"), "Count", StorageResolution.STANDARD), + ("metric", float("nan"), "Count", StorageResolution.STANDARD), + ("metric", math.inf, "Seconds", StorageResolution.STANDARD), + ("metric", -math.inf, "Seconds", StorageResolution.STANDARD), + ("metric", math.nan, "Seconds", StorageResolution.STANDARD), + ("metric", 1, "Kilometers/Fahrenheit", StorageResolution.STANDARD), ("metric", 1, "Seconds", 2), ("metric", 1, "Seconds", None) ] diff --git a/tests/logger/test_metrics_logger.py b/tests/logger/test_metrics_logger.py index 02dbeab..b935a4d 100644 --- a/tests/logger/test_metrics_logger.py +++ b/tests/logger/test_metrics_logger.py @@ -3,6 +3,7 @@ from aws_embedded_metrics.sinks import Sink from aws_embedded_metrics.environment import Environment from aws_embedded_metrics.exceptions import InvalidNamespaceError, InvalidMetricError +from aws_embedded_metrics.storageResolution import StorageResolution import aws_embedded_metrics.constants as constants import pytest from faker import Faker @@ -58,7 +59,7 @@ async def test_can_put_metric_with_different_storage_resolution_different_flush( # arrange expected_key = fake.word() expected_value = fake.random.randrange(100) - metric_storageResolution = 1 + metric_storageResolution = StorageResolution.HIGH logger, sink, env = get_logger_and_sink(mocker) @@ -79,7 +80,7 @@ async def test_can_put_metric_with_different_storage_resolution_different_flush( context = sink.accept.call_args[0][0] assert context.metrics[expected_key].values == [expected_value] assert context.metrics[expected_key].unit == "None" - assert context.metrics[expected_key].storageResolution == 60 + assert context.metrics[expected_key].storageResolution == StorageResolution.STANDARD @pytest.mark.asyncio @@ -91,9 +92,9 @@ async def test_cannot_put_metric_with_different_storage_resolution_same_flush(mo logger, sink, env = get_logger_and_sink(mocker) # act - logger.put_metric(expected_key, expected_value, None, 1) + logger.put_metric(expected_key, expected_value, None, StorageResolution.HIGH) with pytest.raises(InvalidMetricError): - logger.put_metric(expected_key, expected_value, None, 60) + logger.put_metric(expected_key, expected_value, None, StorageResolution.STANDARD) await logger.flush() diff --git a/tests/serializer/test_log_serializer.py b/tests/serializer/test_log_serializer.py index 9abad2c..131c340 100644 --- a/tests/serializer/test_log_serializer.py +++ b/tests/serializer/test_log_serializer.py @@ -2,6 +2,7 @@ from aws_embedded_metrics.exceptions import DimensionSetExceededError from aws_embedded_metrics.logger.metrics_context import MetricsContext from aws_embedded_metrics.serializers.log_serializer import LogSerializer +from aws_embedded_metrics.storageResolution import StorageResolution from collections import Counter from faker import Faker import json @@ -92,7 +93,30 @@ def test_serialize_metrics(): assert_json_equality(result_json, expected) -def test_serialize_metrics_with_storageResolution(): +def test_serialize_metrics_with_Standard_storageResolution(): + # arrange + expected_key = fake.word() + expected_value = fake.random.randrange(0, 100) + + expected_metric_definition = {"Name": expected_key, "Unit": "None"} + + expected = {**get_empty_payload()} + expected[expected_key] = expected_value + expected["_aws"]["CloudWatchMetrics"][0]["Metrics"].append( + expected_metric_definition + ) + + context = get_context() + context.put_metric(expected_key, expected_value, "None", StorageResolution.STANDARD) + + # act + result_json = serializer.serialize(context)[0] + + # assert + assert_json_equality(result_json, expected) + + +def test_serialize_metrics_with_High_storageResolution(): # arrange expected_key = fake.word() expected_value = fake.random.randrange(0, 100) @@ -106,7 +130,7 @@ def test_serialize_metrics_with_storageResolution(): ) context = get_context() - context.put_metric(expected_key, expected_value, "None", 1) + context.put_metric(expected_key, expected_value, "None", StorageResolution.HIGH) # act result_json = serializer.serialize(context)[0] From 68eb153aafc92d37259b911df475ebd728aa3c9b Mon Sep 17 00:00:00 2001 From: Meshwa Savalia <35884424+meshwa19@users.noreply.github.com> Date: Thu, 12 Jan 2023 12:17:09 -0800 Subject: [PATCH 06/12] Update aws_embedded_metrics/validator.py Co-authored-by: Amruth Rayabagi <118546401+rayabagi@users.noreply.github.com> --- aws_embedded_metrics/validator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_embedded_metrics/validator.py b/aws_embedded_metrics/validator.py index abd776b..7f4ec53 100644 --- a/aws_embedded_metrics/validator.py +++ b/aws_embedded_metrics/validator.py @@ -88,7 +88,7 @@ def validate_metric(name: str, value: float, unit: Optional[str], storageResolut raise InvalidMetricError(f"Metric storage Resolution is not valid: {storageResolution}") if metricNameAndResolutionMap and name in metricNameAndResolutionMap: - if metricNameAndResolutionMap.get(name) is not storageResolution: + if if metricNameAndResolutionMap and name in metricNameAndResolutionMap and metricNameAndResolutionMap.get(name) is not storageResolution : raise InvalidMetricError( "Resolution for metrics ${name} is already set. A single log event cannot have a metric with two different resolutions.") From f382031b157bbc5294a01c35a1a584c2e126a073 Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Thu, 12 Jan 2023 12:32:39 -0800 Subject: [PATCH 07/12] Minor fix --- aws_embedded_metrics/validator.py | 7 +++---- tests/integ/agent/test_end_to_end.py | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/aws_embedded_metrics/validator.py b/aws_embedded_metrics/validator.py index 7f4ec53..e88b8be 100644 --- a/aws_embedded_metrics/validator.py +++ b/aws_embedded_metrics/validator.py @@ -87,10 +87,9 @@ def validate_metric(name: str, value: float, unit: Optional[str], storageResolut if storageResolution is None or storageResolution not in StorageResolution: raise InvalidMetricError(f"Metric storage Resolution is not valid: {storageResolution}") - if metricNameAndResolutionMap and name in metricNameAndResolutionMap: - if if metricNameAndResolutionMap and name in metricNameAndResolutionMap and metricNameAndResolutionMap.get(name) is not storageResolution : - raise InvalidMetricError( - "Resolution for metrics ${name} is already set. A single log event cannot have a metric with two different resolutions.") + if metricNameAndResolutionMap and name in metricNameAndResolutionMap and metricNameAndResolutionMap.get(name) is not storageResolution: + raise InvalidMetricError( + "Resolution for metrics ${name} is already set. A single log event cannot have a metric with two different resolutions.") def validate_namespace(namespace: str) -> None: diff --git a/tests/integ/agent/test_end_to_end.py b/tests/integ/agent/test_end_to_end.py index a47fb0f..18fbfd1 100644 --- a/tests/integ/agent/test_end_to_end.py +++ b/tests/integ/agent/test_end_to_end.py @@ -41,7 +41,7 @@ async def test_end_to_end_tcp_multiple_flushes(): @metric_scope async def do_work(metrics): metrics.put_dimensions({"Operation": "Agent"}) - metrics.put_metric(metric_name, 100, "Milliseconds", StorageResolution.HIGH) + metrics.put_metric(metric_name, 100, "Milliseconds") metrics.set_property("RequestId", "422b1569-16f6-4a03-b8f0-fe3fd9b100f8") # act @@ -71,7 +71,7 @@ async def test_end_to_end_udp(): @metric_scope async def do_work(metrics): metrics.put_dimensions({"Operation": "Agent"}) - metrics.put_metric(metric_name, 100, "Milliseconds", StorageResolution.HIGH) + metrics.put_metric(metric_name, 100, "Milliseconds") metrics.set_property("RequestId", "422b1569-16f6-4a03-b8f0-fe3fd9b100f8") # act From 0e17199d2beef50f3356bf01f8ba01805b9c8b93 Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Mon, 16 Jan 2023 13:46:45 -0800 Subject: [PATCH 08/12] Linting fix --- aws_embedded_metrics/validator.py | 2 +- tests/integ/agent/test_end_to_end.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/aws_embedded_metrics/validator.py b/aws_embedded_metrics/validator.py index e88b8be..827d7d8 100644 --- a/aws_embedded_metrics/validator.py +++ b/aws_embedded_metrics/validator.py @@ -87,7 +87,7 @@ def validate_metric(name: str, value: float, unit: Optional[str], storageResolut if storageResolution is None or storageResolution not in StorageResolution: raise InvalidMetricError(f"Metric storage Resolution is not valid: {storageResolution}") - if metricNameAndResolutionMap and name in metricNameAndResolutionMap and metricNameAndResolutionMap.get(name) is not storageResolution: + if metricNameAndResolutionMap and name in metricNameAndResolutionMap and metricNameAndResolutionMap.get(name) is not storageResolution: raise InvalidMetricError( "Resolution for metrics ${name} is already set. A single log event cannot have a metric with two different resolutions.") diff --git a/tests/integ/agent/test_end_to_end.py b/tests/integ/agent/test_end_to_end.py index 18fbfd1..9bb363e 100644 --- a/tests/integ/agent/test_end_to_end.py +++ b/tests/integ/agent/test_end_to_end.py @@ -1,6 +1,5 @@ from aws_embedded_metrics.config import get_config from aws_embedded_metrics import metric_scope -from aws_embedded_metrics.storageResolution import StorageResolution import pytest import boto3 import logging From b13b9806355de77136dd0fbf13427e579eec363b Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Tue, 17 Jan 2023 08:11:46 -0800 Subject: [PATCH 09/12] remove unnecessary checks for MetricMap --- aws_embedded_metrics/logger/metrics_context.py | 3 +-- aws_embedded_metrics/validator.py | 2 +- tests/logger/test_metrics_context.py | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/aws_embedded_metrics/logger/metrics_context.py b/aws_embedded_metrics/logger/metrics_context.py index 6040a2d..7b452f9 100644 --- a/aws_embedded_metrics/logger/metrics_context.py +++ b/aws_embedded_metrics/logger/metrics_context.py @@ -58,8 +58,7 @@ def put_metric(self, key: str, value: float, unit: str = None, storageResolution metric.add_value(value) else: self.metrics[key] = Metric(value, unit, storageResolution) - if key not in self.metricNameAndResolutionMap: - self.metricNameAndResolutionMap[key] = storageResolution + self.metricNameAndResolutionMap[key] = storageResolution def put_dimensions(self, dimension_set: Dict[str, str]) -> None: """ diff --git a/aws_embedded_metrics/validator.py b/aws_embedded_metrics/validator.py index 827d7d8..7093515 100644 --- a/aws_embedded_metrics/validator.py +++ b/aws_embedded_metrics/validator.py @@ -87,7 +87,7 @@ def validate_metric(name: str, value: float, unit: Optional[str], storageResolut if storageResolution is None or storageResolution not in StorageResolution: raise InvalidMetricError(f"Metric storage Resolution is not valid: {storageResolution}") - if metricNameAndResolutionMap and name in metricNameAndResolutionMap and metricNameAndResolutionMap.get(name) is not storageResolution: + if name in metricNameAndResolutionMap and metricNameAndResolutionMap.get(name) is not storageResolution: raise InvalidMetricError( "Resolution for metrics ${name} is already set. A single log event cannot have a metric with two different resolutions.") diff --git a/tests/logger/test_metrics_context.py b/tests/logger/test_metrics_context.py index 7dca3ef..7f8e773 100644 --- a/tests/logger/test_metrics_context.py +++ b/tests/logger/test_metrics_context.py @@ -318,6 +318,7 @@ def test_put_metric_uses_standard_storage_resolution_if_not_provided(): ("metric", math.nan, "Seconds", StorageResolution.STANDARD), ("metric", 1, "Kilometers/Fahrenheit", StorageResolution.STANDARD), ("metric", 1, "Seconds", 2), + ("metric", 1, "Seconds", 0), ("metric", 1, "Seconds", None) ] ) From 389f31e1cad3cce0c85588006e08c5ce16cfab1d Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Mon, 23 Jan 2023 07:08:42 -0800 Subject: [PATCH 10/12] rename storageResolution as per naming convention --- README.md | 15 ++++++++++++--- aws_embedded_metrics/logger/metric.py | 6 +++--- aws_embedded_metrics/logger/metrics_context.py | 12 ++++++------ aws_embedded_metrics/logger/metrics_logger.py | 6 +++--- .../serializers/log_serializer.py | 10 +++++----- ...torageResolution.py => storage_resolution.py} | 0 aws_embedded_metrics/validator.py | 16 ++++++++-------- examples/ec2/app.py | 2 +- examples/lambda/function.py | 2 +- tests/canary/agent/canary.py | 2 +- tests/logger/test_metrics_context.py | 16 ++++++++-------- tests/logger/test_metrics_logger.py | 9 ++++----- tests/serializer/test_log_serializer.py | 6 +++--- 13 files changed, 55 insertions(+), 47 deletions(-) rename aws_embedded_metrics/{storageResolution.py => storage_resolution.py} (100%) diff --git a/README.md b/README.md index 5bf1e5e..5058e43 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,8 @@ from aws_embedded_metrics import metric_scope @metric_scope def my_handler(metrics): metrics.put_dimensions({"Foo": "Bar"}) - metrics.put_metric("ProcessingLatency", 100, "Milliseconds", 60) + metrics.put_metric("ProcessingLatency", 100, "Milliseconds", StorageResolution.STANDARD) + metrics.put_metric("Memory.HeapUsed", 1600424.0, "Bytes", StorageResolution.HIGH) metrics.set_property("AccountId", "123456789012") metrics.set_property("RequestId", "422b1569-16f6-4a03") metrics.set_property("DeviceId", "61270781-c6ac-46f1") @@ -53,7 +54,7 @@ def my_handler(metrics): The `MetricsLogger` is the interface you will use to publish embedded metrics. -- **put_metric**(key: str, value: float, unit: str = "None", storageResolution: int = 60) -> MetricsLogger +- **put_metric**(key: str, value: float, unit: str = "None", storage_resolution: int = 60) -> MetricsLogger Adds a new metric to the current logger context. Multiple metrics using the same key will be appended to an array of values. Multiple metrics cannot have same key and different storage resolution. The Embedded Metric Format supports a maximum of 100 values per key. If more metric values are added than are supported by the format, the logger will be flushed to allow for new metric values to be captured. @@ -64,10 +65,18 @@ Requirements: - Values must be in the range of 8.515920e-109 to 1.174271e+108. In addition, special values (for example, NaN, +Infinity, -Infinity) are not supported. - Metrics must meet CloudWatch Metrics requirements, otherwise a `InvalidMetricError` will be thrown. See [MetricDatum](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_MetricDatum.html) for valid values. +- ##### Storage Resolution +An OPTIONAL value representing the storage resolution for the corresponding metric. Setting this to `High` specifies this metric as a high-resolution metric, so that CloudWatch stores the metric with sub-minute resolution down to one second. Setting this to `Standard` specifies this metric as a standard-resolution metric, which CloudWatch stores at 1-minute resolution. If a value is not provided, then a default value of `Standard` is assumed. See [Cloud Watch High-Resolution metrics](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/publishingMetrics.html#high-resolution-metrics) + Examples: ```py -put_metric("Latency", 200, "Milliseconds", 60) +# Standard Resolution example +put_metric("Latency", 200, "Milliseconds") +put_metric("Latency", 201, "Milliseconds", StorageResolution.STANDARD) + +# High Resolution example +put_metric("Memory.HeapUsed", 1600424.0, Unit.BYTES, StorageResolution.HIGH) ``` - **set_property**(key: str, value: Any) -> MetricsLogger diff --git a/aws_embedded_metrics/logger/metric.py b/aws_embedded_metrics/logger/metric.py index a8adb15..ee02259 100644 --- a/aws_embedded_metrics/logger/metric.py +++ b/aws_embedded_metrics/logger/metric.py @@ -10,14 +10,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from aws_embedded_metrics.storageResolution import StorageResolution +from aws_embedded_metrics.storage_resolution import StorageResolution class Metric(object): - def __init__(self, value: float, unit: str = None, storageResolution: StorageResolution = StorageResolution.STANDARD): + def __init__(self, value: float, unit: str = None, storage_resolution: StorageResolution = StorageResolution.STANDARD): self.values = [value] self.unit = unit or "None" - self.storageResolution = storageResolution or StorageResolution.STANDARD + self.storage_resolution = storage_resolution or StorageResolution.STANDARD def add_value(self, value: float) -> None: self.values.append(value) diff --git a/aws_embedded_metrics/logger/metrics_context.py b/aws_embedded_metrics/logger/metrics_context.py index 7b452f9..3e5303a 100644 --- a/aws_embedded_metrics/logger/metrics_context.py +++ b/aws_embedded_metrics/logger/metrics_context.py @@ -16,7 +16,7 @@ from aws_embedded_metrics.config import get_config from aws_embedded_metrics.logger.metric import Metric from aws_embedded_metrics.validator import validate_dimension_set, validate_metric -from aws_embedded_metrics.storageResolution import StorageResolution +from aws_embedded_metrics.storage_resolution import StorageResolution from typing import List, Dict, Any, Set @@ -40,9 +40,9 @@ def __init__( self.metrics: Dict[str, Metric] = {} self.should_use_default_dimensions = True self.meta: Dict[str, Any] = {"Timestamp": utils.now()} - self.metricNameAndResolutionMap: Dict[str, StorageResolution] = {} + self.metric_name_and_resolution_map: Dict[str, StorageResolution] = {} - def put_metric(self, key: str, value: float, unit: str = None, storageResolution: StorageResolution = StorageResolution.STANDARD) -> None: + def put_metric(self, key: str, value: float, unit: str = None, storage_resolution: StorageResolution = StorageResolution.STANDARD) -> None: """ Adds a metric measurement to the context. Multiple calls using the same key will be stored as an @@ -51,14 +51,14 @@ def put_metric(self, key: str, value: float, unit: str = None, storageResolution context.put_metric("Latency", 100, "Milliseconds") ``` """ - validate_metric(key, value, unit, storageResolution, self.metricNameAndResolutionMap) + validate_metric(key, value, unit, storage_resolution, self.metric_name_and_resolution_map) metric = self.metrics.get(key) if metric: # TODO: we should log a warning if the unit has been changed metric.add_value(value) else: - self.metrics[key] = Metric(value, unit, storageResolution) - self.metricNameAndResolutionMap[key] = storageResolution + self.metrics[key] = Metric(value, unit, storage_resolution) + self.metric_name_and_resolution_map[key] = storage_resolution def put_dimensions(self, dimension_set: Dict[str, str]) -> None: """ diff --git a/aws_embedded_metrics/logger/metrics_logger.py b/aws_embedded_metrics/logger/metrics_logger.py index c89972f..c25cae2 100644 --- a/aws_embedded_metrics/logger/metrics_logger.py +++ b/aws_embedded_metrics/logger/metrics_logger.py @@ -15,7 +15,7 @@ from aws_embedded_metrics.logger.metrics_context import MetricsContext from aws_embedded_metrics.validator import validate_namespace from aws_embedded_metrics.config import get_config -from aws_embedded_metrics.storageResolution import StorageResolution +from aws_embedded_metrics.storage_resolution import StorageResolution from typing import Any, Awaitable, Callable, Dict, Tuple import sys import traceback @@ -80,9 +80,9 @@ def set_namespace(self, namespace: str) -> "MetricsLogger": return self def put_metric( - self, key: str, value: float, unit: str = "None", storageResolution: StorageResolution = StorageResolution.STANDARD + self, key: str, value: float, unit: str = "None", storage_resolution: StorageResolution = StorageResolution.STANDARD ) -> "MetricsLogger": - self.context.put_metric(key, value, unit, storageResolution) + self.context.put_metric(key, value, unit, storage_resolution) return self def add_stack_trace(self, key: str, details: Any = None, exc_info: Tuple = None) -> "MetricsLogger": diff --git a/aws_embedded_metrics/serializers/log_serializer.py b/aws_embedded_metrics/serializers/log_serializer.py index e63e4b8..cf8e05a 100644 --- a/aws_embedded_metrics/serializers/log_serializer.py +++ b/aws_embedded_metrics/serializers/log_serializer.py @@ -18,7 +18,7 @@ MAX_DIMENSION_SET_SIZE, MAX_METRICS_PER_EVENT, MAX_DATAPOINTS_PER_METRIC ) from aws_embedded_metrics.exceptions import DimensionSetExceededError -from aws_embedded_metrics.storageResolution import StorageResolution +from aws_embedded_metrics.storage_resolution import StorageResolution import json from typing import Any, Dict, List @@ -88,11 +88,11 @@ def create_body() -> Dict[str, Any]: if len(metric.values) > end_index: remaining_data = True - metricBody = {"Name": metric_name, "Unit": metric.unit} - if metric.storageResolution == StorageResolution.HIGH: - metricBody["StorageResolution"] = metric.storageResolution.value # type: ignore + metric_body = {"Name": metric_name, "Unit": metric.unit} + if metric.storage_resolution == StorageResolution.HIGH: + metric_body["StorageResolution"] = metric.storage_resolution.value # type: ignore if not config.disable_metric_extraction: - current_body["_aws"]["CloudWatchMetrics"][0]["Metrics"].append(metricBody) + current_body["_aws"]["CloudWatchMetrics"][0]["Metrics"].append(metric_body) num_metrics_in_current_body += 1 if (num_metrics_in_current_body == MAX_METRICS_PER_EVENT): diff --git a/aws_embedded_metrics/storageResolution.py b/aws_embedded_metrics/storage_resolution.py similarity index 100% rename from aws_embedded_metrics/storageResolution.py rename to aws_embedded_metrics/storage_resolution.py diff --git a/aws_embedded_metrics/validator.py b/aws_embedded_metrics/validator.py index 7093515..ac30f8e 100644 --- a/aws_embedded_metrics/validator.py +++ b/aws_embedded_metrics/validator.py @@ -15,7 +15,7 @@ import re from typing import Dict, Optional from aws_embedded_metrics.unit import Unit -from aws_embedded_metrics.storageResolution import StorageResolution +from aws_embedded_metrics.storage_resolution import StorageResolution from aws_embedded_metrics.exceptions import DimensionSetExceededError, InvalidDimensionError, InvalidMetricError, InvalidNamespaceError import aws_embedded_metrics.constants as constants @@ -58,7 +58,7 @@ def validate_dimension_set(dimension_set: Dict[str, str]) -> None: raise InvalidDimensionError("Dimension name cannot start with ':'") -def validate_metric(name: str, value: float, unit: Optional[str], storageResolution: StorageResolution, metricNameAndResolutionMap: dict) -> None: # noqa: E501 +def validate_metric(name: str, value: float, unit: Optional[str], storage_resolution: StorageResolution, metric_name_and_resolution_map: dict) -> None: # noqa: E501 """ Validates a metric @@ -66,8 +66,8 @@ def validate_metric(name: str, value: float, unit: Optional[str], storageResolut name (str): The name of the metric value (float): The value of the metric unit (Optional[str]): The unit of the metric - storageResolution (Optional[int]): The storage resolution of metric - metricNameAndResolutionMap (dict): The map used for validating metric + storage_resolution (Optional[int]): The storage resolution of metric + metric_name_and_resolution_map (dict): The map used for validating metric Raises: InvalidMetricError: If the metric is invalid @@ -84,12 +84,12 @@ def validate_metric(name: str, value: float, unit: Optional[str], storageResolut if unit is not None and unit not in Unit: raise InvalidMetricError(f"Metric unit is not valid: {unit}") - if storageResolution is None or storageResolution not in StorageResolution: - raise InvalidMetricError(f"Metric storage Resolution is not valid: {storageResolution}") + if storage_resolution is None or storage_resolution not in StorageResolution: + raise InvalidMetricError(f"Metric storage resolution is not valid: {storage_resolution}") - if name in metricNameAndResolutionMap and metricNameAndResolutionMap.get(name) is not storageResolution: + if name in metric_name_and_resolution_map and metric_name_and_resolution_map.get(name) is not storage_resolution: raise InvalidMetricError( - "Resolution for metrics ${name} is already set. A single log event cannot have a metric with two different resolutions.") + f"Resolution for metrics {name} is already set. A single log event cannot have a metric with two different resolutions.") def validate_namespace(namespace: str) -> None: diff --git a/examples/ec2/app.py b/examples/ec2/app.py index 46d38ee..00a587d 100644 --- a/examples/ec2/app.py +++ b/examples/ec2/app.py @@ -1,5 +1,5 @@ from aws_embedded_metrics import metric_scope -from aws_embedded_metrics.storageResolution import StorageResolution +from aws_embedded_metrics.storage_resolution import StorageResolution import logging diff --git a/examples/lambda/function.py b/examples/lambda/function.py index a1b6657..3c9801f 100644 --- a/examples/lambda/function.py +++ b/examples/lambda/function.py @@ -1,5 +1,5 @@ from aws_embedded_metrics import metric_scope -from aws_embedded_metrics.storageResolution import StorageResolution +from aws_embedded_metrics.storage_resolution import StorageResolution @metric_scope diff --git a/tests/canary/agent/canary.py b/tests/canary/agent/canary.py index 7e9113e..7bbf807 100644 --- a/tests/canary/agent/canary.py +++ b/tests/canary/agent/canary.py @@ -2,7 +2,7 @@ import aws_embedded_metrics from aws_embedded_metrics import metric_scope from aws_embedded_metrics.config import get_config -from aws_embedded_metrics.storageResolution import StorageResolution +from aws_embedded_metrics.storage_resolution import StorageResolution from getversion import get_module_version import os import psutil diff --git a/tests/logger/test_metrics_context.py b/tests/logger/test_metrics_context.py index 7f8e773..86f62d0 100644 --- a/tests/logger/test_metrics_context.py +++ b/tests/logger/test_metrics_context.py @@ -3,7 +3,7 @@ import random from aws_embedded_metrics import constants from aws_embedded_metrics.unit import Unit -from aws_embedded_metrics.storageResolution import StorageResolution +from aws_embedded_metrics.storage_resolution import StorageResolution from aws_embedded_metrics import config from aws_embedded_metrics.logger.metrics_context import MetricsContext from aws_embedded_metrics.constants import DEFAULT_NAMESPACE @@ -264,16 +264,16 @@ def test_put_metric_adds_metrics(): metric_key = fake.word() metric_value = fake.random.random() metric_unit = random.choice(list(Unit)).value - metric_storageResolution = random.choice(list(StorageResolution)).value + metric_storage_resolution = random.choice(list(StorageResolution)).value # act - context.put_metric(metric_key, metric_value, metric_unit, metric_storageResolution) + context.put_metric(metric_key, metric_value, metric_unit, metric_storage_resolution) # assert metric = context.metrics[metric_key] assert metric.unit == metric_unit assert metric.values == [metric_value] - assert metric.storageResolution == metric_storageResolution + assert metric.storage_resolution == metric_storage_resolution def test_put_metric_uses_none_unit_if_not_provided(): @@ -301,11 +301,11 @@ def test_put_metric_uses_standard_storage_resolution_if_not_provided(): # assert metric = context.metrics[metric_key] - assert metric.storageResolution == StorageResolution.STANDARD + assert metric.storage_resolution == StorageResolution.STANDARD @pytest.mark.parametrize( - "name, value, unit, storageResolution", + "name, value, unit, storage_resolution", [ ("", 1, "None", StorageResolution.STANDARD), (" ", 1, "Seconds", StorageResolution.STANDARD), @@ -322,11 +322,11 @@ def test_put_metric_uses_standard_storage_resolution_if_not_provided(): ("metric", 1, "Seconds", None) ] ) -def test_put_invalid_metric_raises_exception(name, value, unit, storageResolution): +def test_put_invalid_metric_raises_exception(name, value, unit, storage_resolution): context = MetricsContext() with pytest.raises(InvalidMetricError): - context.put_metric(name, value, unit, storageResolution) + context.put_metric(name, value, unit, storage_resolution) def test_create_copy_with_context_creates_new_instance(): diff --git a/tests/logger/test_metrics_logger.py b/tests/logger/test_metrics_logger.py index b935a4d..8602c47 100644 --- a/tests/logger/test_metrics_logger.py +++ b/tests/logger/test_metrics_logger.py @@ -3,7 +3,7 @@ from aws_embedded_metrics.sinks import Sink from aws_embedded_metrics.environment import Environment from aws_embedded_metrics.exceptions import InvalidNamespaceError, InvalidMetricError -from aws_embedded_metrics.storageResolution import StorageResolution +from aws_embedded_metrics.storage_resolution import StorageResolution import aws_embedded_metrics.constants as constants import pytest from faker import Faker @@ -59,19 +59,18 @@ async def test_can_put_metric_with_different_storage_resolution_different_flush( # arrange expected_key = fake.word() expected_value = fake.random.randrange(100) - metric_storageResolution = StorageResolution.HIGH logger, sink, env = get_logger_and_sink(mocker) # act - logger.put_metric(expected_key, expected_value, None, metric_storageResolution) + logger.put_metric(expected_key, expected_value, None, StorageResolution.HIGH) await logger.flush() # assert context = sink.accept.call_args[0][0] assert context.metrics[expected_key].values == [expected_value] assert context.metrics[expected_key].unit == "None" - assert context.metrics[expected_key].storageResolution == metric_storageResolution + assert context.metrics[expected_key].storage_resolution == StorageResolution.HIGH expected_key = fake.word() expected_value = fake.random.randrange(100) @@ -80,7 +79,7 @@ async def test_can_put_metric_with_different_storage_resolution_different_flush( context = sink.accept.call_args[0][0] assert context.metrics[expected_key].values == [expected_value] assert context.metrics[expected_key].unit == "None" - assert context.metrics[expected_key].storageResolution == StorageResolution.STANDARD + assert context.metrics[expected_key].storage_resolution == StorageResolution.STANDARD @pytest.mark.asyncio diff --git a/tests/serializer/test_log_serializer.py b/tests/serializer/test_log_serializer.py index 131c340..60d1e63 100644 --- a/tests/serializer/test_log_serializer.py +++ b/tests/serializer/test_log_serializer.py @@ -2,7 +2,7 @@ from aws_embedded_metrics.exceptions import DimensionSetExceededError from aws_embedded_metrics.logger.metrics_context import MetricsContext from aws_embedded_metrics.serializers.log_serializer import LogSerializer -from aws_embedded_metrics.storageResolution import StorageResolution +from aws_embedded_metrics.storage_resolution import StorageResolution from collections import Counter from faker import Faker import json @@ -93,7 +93,7 @@ def test_serialize_metrics(): assert_json_equality(result_json, expected) -def test_serialize_metrics_with_Standard_storageResolution(): +def test_serialize_metrics_with_standard_storage_resolution(): # arrange expected_key = fake.word() expected_value = fake.random.randrange(0, 100) @@ -116,7 +116,7 @@ def test_serialize_metrics_with_Standard_storageResolution(): assert_json_equality(result_json, expected) -def test_serialize_metrics_with_High_storageResolution(): +def test_serialize_metrics_with_high_storage_resolution(): # arrange expected_key = fake.word() expected_value = fake.random.randrange(0, 100) From 009e75230cd6e6fb0b30a6235d825f1c5698f5c2 Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Mon, 23 Jan 2023 07:08:42 -0800 Subject: [PATCH 11/12] rename storageResolution as per naming convention --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5058e43..561bb7d 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ put_metric("Latency", 200, "Milliseconds") put_metric("Latency", 201, "Milliseconds", StorageResolution.STANDARD) # High Resolution example -put_metric("Memory.HeapUsed", 1600424.0, Unit.BYTES, StorageResolution.HIGH) +put_metric("Memory.HeapUsed", 1600424.0, "Bytes", StorageResolution.HIGH) ``` - **set_property**(key: str, value: Any) -> MetricsLogger From e516a848585804cee36b131b2dcece220cd7f86f Mon Sep 17 00:00:00 2001 From: Meshwa Savalia Date: Mon, 23 Jan 2023 11:29:40 -0800 Subject: [PATCH 12/12] minor fix --- aws_embedded_metrics/validator.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/aws_embedded_metrics/validator.py b/aws_embedded_metrics/validator.py index ac30f8e..21f9c41 100644 --- a/aws_embedded_metrics/validator.py +++ b/aws_embedded_metrics/validator.py @@ -58,7 +58,11 @@ def validate_dimension_set(dimension_set: Dict[str, str]) -> None: raise InvalidDimensionError("Dimension name cannot start with ':'") -def validate_metric(name: str, value: float, unit: Optional[str], storage_resolution: StorageResolution, metric_name_and_resolution_map: dict) -> None: # noqa: E501 +def validate_metric(name: str, + value: float, + unit: Optional[str], + storage_resolution: StorageResolution, + metric_name_and_resolution_map: dict) -> None: """ Validates a metric