From 4c36304abcfb3f9e22a7c48c8c896163fd4ac51d Mon Sep 17 00:00:00 2001 From: Amruth Rayabagi <118546401+rayabagi@users.noreply.github.com> Date: Wed, 13 Sep 2023 14:47:43 -0700 Subject: [PATCH] Added support to set custom timestamp (#110) Added support to set custom timestamp Co-authored-by: Mark Kuhn --- README.md | 14 ++++++ aws_embedded_metrics/constants.py | 3 ++ aws_embedded_metrics/exceptions.py | 6 +++ .../logger/metrics_context.py | 23 ++++++++- aws_embedded_metrics/logger/metrics_logger.py | 5 ++ aws_embedded_metrics/utils.py | 8 ++++ aws_embedded_metrics/validator.py | 33 ++++++++++++- setup.py | 2 +- tests/integ/agent/test_end_to_end.py | 1 + tests/logger/test_metrics_context.py | 48 +++++++++++++++++-- tests/logger/test_metrics_logger.py | 18 ++++++- 11 files changed, 152 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 199161f..1903169 100644 --- a/README.md +++ b/README.md @@ -178,6 +178,20 @@ Examples: set_namespace("MyApplication") ``` +- **set_timestamp**(timestamp: datetime) -> MetricsLogger + +Sets the timestamp of the metrics. If not set, current time of the client will be used. + +Timestamp must meet CloudWatch requirements, otherwise a InvalidTimestampError will be thrown. See [Timestamps](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#about_timestamp) for valid values. + +Examples: + +```py + set_timestamp(datetime.datetime.now()) +``` + + + - **flush**() Flushes the current MetricsContext to the configured sink and resets all properties and metric values. The namespace and default dimensions will be preserved across flushes. diff --git a/aws_embedded_metrics/constants.py b/aws_embedded_metrics/constants.py index af2ac16..45f1b6a 100644 --- a/aws_embedded_metrics/constants.py +++ b/aws_embedded_metrics/constants.py @@ -20,3 +20,6 @@ MAX_METRIC_NAME_LENGTH = 1024 MAX_NAMESPACE_LENGTH = 256 VALID_NAMESPACE_REGEX = '^[a-zA-Z0-9._#:/-]+$' +TIMESTAMP = "Timestamp" +MAX_TIMESTAMP_PAST_AGE = 14 * 24 * 60 * 60 * 1000 # 14 days +MAX_TIMESTAMP_FUTURE_AGE = 2 * 60 * 60 * 1000 # 2 hours diff --git a/aws_embedded_metrics/exceptions.py b/aws_embedded_metrics/exceptions.py index 2ca0f8d..22cfe94 100644 --- a/aws_embedded_metrics/exceptions.py +++ b/aws_embedded_metrics/exceptions.py @@ -33,3 +33,9 @@ class InvalidNamespaceError(Exception): def __init__(self, message: str) -> None: # Call the base class constructor with the parameters it needs super().__init__(message) + + +class InvalidTimestampError(Exception): + def __init__(self, message: str) -> None: + # Call the base class constructor with the parameters it needs + super().__init__(message) diff --git a/aws_embedded_metrics/logger/metrics_context.py b/aws_embedded_metrics/logger/metrics_context.py index 3e5303a..ac11ea9 100644 --- a/aws_embedded_metrics/logger/metrics_context.py +++ b/aws_embedded_metrics/logger/metrics_context.py @@ -12,7 +12,8 @@ # limitations under the License. -from aws_embedded_metrics import constants, utils +from datetime import datetime +from aws_embedded_metrics import constants, utils, validator from aws_embedded_metrics.config import get_config from aws_embedded_metrics.logger.metric import Metric from aws_embedded_metrics.validator import validate_dimension_set, validate_metric @@ -39,7 +40,7 @@ def __init__( self.default_dimensions: Dict[str, str] = default_dimensions or {} self.metrics: Dict[str, Metric] = {} self.should_use_default_dimensions = True - self.meta: Dict[str, Any] = {"Timestamp": utils.now()} + self.meta: Dict[str, Any] = {constants.TIMESTAMP: utils.now()} self.metric_name_and_resolution_map: Dict[str, StorageResolution] = {} def put_metric(self, key: str, value: float, unit: str = None, storage_resolution: StorageResolution = StorageResolution.STANDARD) -> None: @@ -176,3 +177,21 @@ def create_copy_with_context(self, preserve_dimensions: bool = False) -> "Metric @staticmethod def empty() -> "MetricsContext": return MetricsContext() + + def set_timestamp(self, timestamp: datetime) -> None: + """ + Set the timestamp of metrics emitted in this context. If not set, the timestamp will default to the time the context is constructed. + + Timestamp must meet CloudWatch requirements, otherwise a InvalidTimestampError will be thrown. + See [Timestamps](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#about_timestamp) + for valid values. + + Parameters: + timestamp (datetime): The timestamp value to be set. + + Raises: + InvalidTimestampError: If the provided timestamp is invalid. + + """ + validator.validate_timestamp(timestamp) + self.meta[constants.TIMESTAMP] = utils.convert_to_milliseconds(timestamp) diff --git a/aws_embedded_metrics/logger/metrics_logger.py b/aws_embedded_metrics/logger/metrics_logger.py index c25cae2..ebbe469 100644 --- a/aws_embedded_metrics/logger/metrics_logger.py +++ b/aws_embedded_metrics/logger/metrics_logger.py @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from datetime import datetime from aws_embedded_metrics.environment import Environment from aws_embedded_metrics.logger.metrics_context import MetricsContext from aws_embedded_metrics.validator import validate_namespace @@ -114,6 +115,10 @@ def add_stack_trace(self, key: str, details: Any = None, exc_info: Tuple = None) self.set_property(key, trace_value) return self + def set_timestamp(self, timestamp: datetime) -> "MetricsLogger": + self.context.set_timestamp(timestamp) + return self + def new(self) -> "MetricsLogger": return MetricsLogger( self.resolve_environment, self.context.create_copy_with_context() diff --git a/aws_embedded_metrics/utils.py b/aws_embedded_metrics/utils.py index b73d10f..8753cd6 100644 --- a/aws_embedded_metrics/utils.py +++ b/aws_embedded_metrics/utils.py @@ -12,4 +12,12 @@ # limitations under the License. import time +from datetime import datetime def now() -> int: return int(round(time.time() * 1000)) + + +def convert_to_milliseconds(dt: datetime) -> int: + if dt == datetime.min: + return 0 + + return int(round(dt.timestamp() * 1000)) diff --git a/aws_embedded_metrics/validator.py b/aws_embedded_metrics/validator.py index 21f9c41..d6ac3fe 100644 --- a/aws_embedded_metrics/validator.py +++ b/aws_embedded_metrics/validator.py @@ -17,7 +17,9 @@ from aws_embedded_metrics.unit import Unit from aws_embedded_metrics.storage_resolution import StorageResolution from aws_embedded_metrics.exceptions import DimensionSetExceededError, InvalidDimensionError, InvalidMetricError, InvalidNamespaceError -import aws_embedded_metrics.constants as constants +from aws_embedded_metrics.exceptions import InvalidTimestampError +from datetime import datetime +from aws_embedded_metrics import constants, utils def validate_dimension_set(dimension_set: Dict[str, str]) -> None: @@ -114,3 +116,32 @@ def validate_namespace(namespace: str) -> None: if not re.match(constants.VALID_NAMESPACE_REGEX, namespace): raise InvalidNamespaceError(f"Namespace contains invalid characters: {namespace}") + + +def validate_timestamp(timestamp: datetime) -> None: + """ + Validates a given timestamp based on CloudWatch Timestamp guidelines. + + Timestamp must meet CloudWatch requirements, otherwise a InvalidTimestampError will be thrown. + See [Timestamps](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#about_timestamp) + for valid values. + + Parameters: + timestamp (datetime): Datetime object representing the timestamp to validate. + + Raises: + InvalidTimestampError: If the timestamp is either None, too old, or too far in the future. + """ + if not timestamp: + raise InvalidTimestampError("Timestamp must be a valid datetime object") + + given_time_in_milliseconds = utils.convert_to_milliseconds(timestamp) + current_time_in_milliseconds = utils.now() + + if given_time_in_milliseconds < (current_time_in_milliseconds - constants.MAX_TIMESTAMP_PAST_AGE): + raise InvalidTimestampError( + f"Timestamp {str(timestamp)} must not be older than {int(constants.MAX_TIMESTAMP_PAST_AGE/(24 * 60 * 60 * 1000))} days") + + if given_time_in_milliseconds > (current_time_in_milliseconds + constants.MAX_TIMESTAMP_FUTURE_AGE): + raise InvalidTimestampError( + f"Timestamp {str(timestamp)} must not be newer than {int(constants.MAX_TIMESTAMP_FUTURE_AGE/(60 * 60 * 1000))} hours") diff --git a/setup.py b/setup.py index 43e81e9..27c3417 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="aws-embedded-metrics", - version="3.1.1", + version="3.2.0", author="Amazon Web Services", author_email="jarnance@amazon.com", description="AWS Embedded Metrics Package", diff --git a/tests/integ/agent/test_end_to_end.py b/tests/integ/agent/test_end_to_end.py index 9bb363e..8b45d39 100644 --- a/tests/integ/agent/test_end_to_end.py +++ b/tests/integ/agent/test_end_to_end.py @@ -42,6 +42,7 @@ async def do_work(metrics): metrics.put_dimensions({"Operation": "Agent"}) metrics.put_metric(metric_name, 100, "Milliseconds") metrics.set_property("RequestId", "422b1569-16f6-4a03-b8f0-fe3fd9b100f8") + metrics.set_timestamp(datetime.utcnow()) # act await do_work() diff --git a/tests/logger/test_metrics_context.py b/tests/logger/test_metrics_context.py index 86f62d0..a0ab133 100644 --- a/tests/logger/test_metrics_context.py +++ b/tests/logger/test_metrics_context.py @@ -1,15 +1,17 @@ +from faker import Faker +from importlib import reload +from datetime import datetime, timedelta import pytest import math import random -from aws_embedded_metrics import constants +from aws_embedded_metrics import constants, utils from aws_embedded_metrics.unit import Unit from aws_embedded_metrics.storage_resolution import StorageResolution from aws_embedded_metrics import config from aws_embedded_metrics.logger.metrics_context import MetricsContext -from aws_embedded_metrics.constants import DEFAULT_NAMESPACE +from aws_embedded_metrics.constants import DEFAULT_NAMESPACE, MAX_TIMESTAMP_FUTURE_AGE, MAX_TIMESTAMP_PAST_AGE from aws_embedded_metrics.exceptions import DimensionSetExceededError, InvalidDimensionError, InvalidMetricError -from importlib import reload -from faker import Faker +from aws_embedded_metrics.exceptions import InvalidTimestampError fake = Faker() @@ -458,6 +460,44 @@ def test_cannot_put_more_than_30_dimensions(): context.put_dimensions(dimension_set) +@pytest.mark.parametrize( + "timestamp", + [ + datetime.now(), + datetime.now() - timedelta(milliseconds=MAX_TIMESTAMP_PAST_AGE - 5000), + datetime.now() + timedelta(milliseconds=MAX_TIMESTAMP_FUTURE_AGE - 5000) + ] +) +def test_set_valid_timestamp_verify_timestamp(timestamp: datetime): + context = MetricsContext() + + context.set_timestamp(timestamp) + + assert context.meta[constants.TIMESTAMP] == utils.convert_to_milliseconds(timestamp) + + +@pytest.mark.parametrize( + "timestamp", + [ + None, + datetime.min, + datetime(1970, 1, 1, 0, 0, 0), + datetime.max, + datetime(9999, 12, 31, 23, 59, 59, 999999), + datetime(1, 1, 1, 0, 0, 0, 0, None), + datetime(1, 1, 1), + datetime(1, 1, 1, 0, 0), + datetime.now() - timedelta(milliseconds=MAX_TIMESTAMP_PAST_AGE + 1), + datetime.now() + timedelta(milliseconds=MAX_TIMESTAMP_FUTURE_AGE + 5000) + ] +) +def test_set_invalid_timestamp_raises_exception(timestamp: datetime): + context = MetricsContext() + + with pytest.raises(InvalidTimestampError): + context.set_timestamp(timestamp) + + # Test utility method diff --git a/tests/logger/test_metrics_logger.py b/tests/logger/test_metrics_logger.py index 8602c47..08bd971 100644 --- a/tests/logger/test_metrics_logger.py +++ b/tests/logger/test_metrics_logger.py @@ -1,4 +1,5 @@ -from aws_embedded_metrics import config +from datetime import datetime +from aws_embedded_metrics import config, utils from aws_embedded_metrics.logger import metrics_logger from aws_embedded_metrics.sinks import Sink from aws_embedded_metrics.environment import Environment @@ -493,6 +494,21 @@ async def test_configure_flush_to_preserve_dimensions(mocker): assert dimensions[0][dimension_key] == dimension_value +@pytest.mark.asyncio +async def test_can_set_timestamp(mocker): + # arrange + expected_value = datetime.now() + + logger, sink, env = get_logger_and_sink(mocker) + + # act + logger.set_timestamp(expected_value) + await logger.flush() + + # assert + context = get_flushed_context(sink) + assert context.meta[constants.TIMESTAMP] == utils.convert_to_milliseconds(expected_value) + # Test helper methods