From c5deb5d2035db978643f36f1f0c07be9a1310c77 Mon Sep 17 00:00:00 2001 From: Amruth Rayabagi Date: Fri, 8 Sep 2023 17:32:13 -0700 Subject: [PATCH 01/18] Added support set custom timestamp --- README.md | 14 +++++++ aws_embedded_metrics/constants.py | 3 ++ aws_embedded_metrics/exceptions.py | 6 +++ .../logger/metrics_context.py | 21 +++++++++- aws_embedded_metrics/utils.py | 5 +++ aws_embedded_metrics/validator.py | 33 +++++++++++++++- tests/logger/test_metrics_context.py | 39 +++++++++++++++++-- 7 files changed, 116 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 199161f..1903169 100644 --- a/README.md +++ b/README.md @@ -178,6 +178,20 @@ Examples: set_namespace("MyApplication") ``` +- **set_timestamp**(timestamp: datetime) -> MetricsLogger + +Sets the timestamp of the metrics. If not set, current time of the client will be used. + +Timestamp must meet CloudWatch requirements, otherwise a InvalidTimestampError will be thrown. See [Timestamps](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#about_timestamp) for valid values. + +Examples: + +```py + set_timestamp(datetime.datetime.now()) +``` + + + - **flush**() Flushes the current MetricsContext to the configured sink and resets all properties and metric values. The namespace and default dimensions will be preserved across flushes. diff --git a/aws_embedded_metrics/constants.py b/aws_embedded_metrics/constants.py index af2ac16..590a1fa 100644 --- a/aws_embedded_metrics/constants.py +++ b/aws_embedded_metrics/constants.py @@ -20,3 +20,6 @@ MAX_METRIC_NAME_LENGTH = 1024 MAX_NAMESPACE_LENGTH = 256 VALID_NAMESPACE_REGEX = '^[a-zA-Z0-9._#:/-]+$' +TIMESTAMP = "Timestamp" +MAX_TIMESTAMP_PAST_AGE_SECONDS = 14 * 24 * 60 * 60 * 1000 # 14 days +MAX_TIMESTAMP_FUTURE_AGE_SECONDS = 2 * 60 * 60 * 1000 # 2 hours diff --git a/aws_embedded_metrics/exceptions.py b/aws_embedded_metrics/exceptions.py index 2ca0f8d..22cfe94 100644 --- a/aws_embedded_metrics/exceptions.py +++ b/aws_embedded_metrics/exceptions.py @@ -33,3 +33,9 @@ class InvalidNamespaceError(Exception): def __init__(self, message: str) -> None: # Call the base class constructor with the parameters it needs super().__init__(message) + + +class InvalidTimestampError(Exception): + def __init__(self, message: str) -> None: + # Call the base class constructor with the parameters it needs + super().__init__(message) diff --git a/aws_embedded_metrics/logger/metrics_context.py b/aws_embedded_metrics/logger/metrics_context.py index 3e5303a..2aba768 100644 --- a/aws_embedded_metrics/logger/metrics_context.py +++ b/aws_embedded_metrics/logger/metrics_context.py @@ -12,7 +12,9 @@ # limitations under the License. +import datetime from aws_embedded_metrics import constants, utils +from aws_embedded_metrics import validator from aws_embedded_metrics.config import get_config from aws_embedded_metrics.logger.metric import Metric from aws_embedded_metrics.validator import validate_dimension_set, validate_metric @@ -39,7 +41,7 @@ def __init__( self.default_dimensions: Dict[str, str] = default_dimensions or {} self.metrics: Dict[str, Metric] = {} self.should_use_default_dimensions = True - self.meta: Dict[str, Any] = {"Timestamp": utils.now()} + self.meta: Dict[str, Any] = {constants.TIMESTAMP: utils.now()} self.metric_name_and_resolution_map: Dict[str, StorageResolution] = {} def put_metric(self, key: str, value: float, unit: str = None, storage_resolution: StorageResolution = StorageResolution.STANDARD) -> None: @@ -176,3 +178,20 @@ def create_copy_with_context(self, preserve_dimensions: bool = False) -> "Metric @staticmethod def empty() -> "MetricsContext": return MetricsContext() + + def set_timestamp(self, timestamp: datetime) -> None: + """ + Update timestamp field in the metadata + + * @see CloudWatch + * Timestamp + + Parameters: + timestamp (datetime): timestamp value to be set + + Raises: + InvalidTimestampError: if timestamp is invalid + """ + validator.validate_timestamp(timestamp) + self.meta[constants.TIMESTAMP] = utils.convert_to_milliseconds(timestamp) diff --git a/aws_embedded_metrics/utils.py b/aws_embedded_metrics/utils.py index b73d10f..a76c2a7 100644 --- a/aws_embedded_metrics/utils.py +++ b/aws_embedded_metrics/utils.py @@ -12,4 +12,9 @@ # limitations under the License. import time +from datetime import datetime def now() -> int: return int(round(time.time() * 1000)) + + +def convert_to_milliseconds(datetime: datetime) -> int: + return int(round(datetime.timestamp() * 1000)) diff --git a/aws_embedded_metrics/validator.py b/aws_embedded_metrics/validator.py index 21f9c41..8712006 100644 --- a/aws_embedded_metrics/validator.py +++ b/aws_embedded_metrics/validator.py @@ -16,8 +16,10 @@ from typing import Dict, Optional from aws_embedded_metrics.unit import Unit from aws_embedded_metrics.storage_resolution import StorageResolution -from aws_embedded_metrics.exceptions import DimensionSetExceededError, InvalidDimensionError, InvalidMetricError, InvalidNamespaceError +from aws_embedded_metrics.exceptions import DimensionSetExceededError, InvalidDimensionError, InvalidMetricError, InvalidNamespaceError, InvalidTimestampError import aws_embedded_metrics.constants as constants +import datetime +from aws_embedded_metrics import utils, constants def validate_dimension_set(dimension_set: Dict[str, str]) -> None: @@ -114,3 +116,32 @@ def validate_namespace(namespace: str) -> None: if not re.match(constants.VALID_NAMESPACE_REGEX, namespace): raise InvalidNamespaceError(f"Namespace contains invalid characters: {namespace}") + + +def validate_timestamp(timestamp: datetime) -> None: + """ + Validates a timestamp + + * @see CloudWatch + * Timestamp + + Parameters: + timestamp (datetime): datetime to validate + + Raises: + InvalidTimestampError: if datetime is invalid + """ + if (timestamp is None): + raise InvalidTimestampError(f"Timestamp cannot be none") + + given_time_in_milliseconds = utils.convert_to_milliseconds(timestamp) + current_time_in_milliseconds = utils.convert_to_milliseconds(datetime.datetime.now()) + + if (given_time_in_milliseconds <= (current_time_in_milliseconds - constants.MAX_TIMESTAMP_PAST_AGE_SECONDS)): + raise InvalidTimestampError( + f"Timestamp {str(timestamp)} must not be older than {str(constants.MAX_TIMESTAMP_PAST_AGE_SECONDS)} milliseconds") + + if (given_time_in_milliseconds >= (current_time_in_milliseconds + constants.MAX_TIMESTAMP_FUTURE_AGE_SECONDS)): + raise InvalidTimestampError( + f"Timestamp {str(timestamp)} must not be newer than {str(constants.MAX_TIMESTAMP_FUTURE_AGE_SECONDS)} milliseconds") diff --git a/tests/logger/test_metrics_context.py b/tests/logger/test_metrics_context.py index 86f62d0..b5327d8 100644 --- a/tests/logger/test_metrics_context.py +++ b/tests/logger/test_metrics_context.py @@ -1,13 +1,14 @@ +import datetime import pytest import math import random -from aws_embedded_metrics import constants +from aws_embedded_metrics import constants, utils from aws_embedded_metrics.unit import Unit from aws_embedded_metrics.storage_resolution import StorageResolution from aws_embedded_metrics import config from aws_embedded_metrics.logger.metrics_context import MetricsContext -from aws_embedded_metrics.constants import DEFAULT_NAMESPACE -from aws_embedded_metrics.exceptions import DimensionSetExceededError, InvalidDimensionError, InvalidMetricError +from aws_embedded_metrics.constants import DEFAULT_NAMESPACE, MAX_TIMESTAMP_FUTURE_AGE_SECONDS, MAX_TIMESTAMP_PAST_AGE_SECONDS +from aws_embedded_metrics.exceptions import DimensionSetExceededError, InvalidDimensionError, InvalidMetricError, InvalidTimestampError from importlib import reload from faker import Faker @@ -467,3 +468,35 @@ def generate_dimension_set(dimensions_to_add): dimension_set[f"{i}"] = fake.word() return dimension_set + + +def test_set_timestamp_verify_timestamp(): + context = MetricsContext() + context.put_metric("TestMetric", 0) + + now = datetime.datetime.now() + context.set_timestamp(now) + + assert context.meta[constants.TIMESTAMP] == utils.convert_to_milliseconds(now) + + +def test_set_timestamp_null_raise_exception(): + context = MetricsContext() + past_date = None + with pytest.raises(InvalidTimestampError): + context.set_timestamp(past_date) + + +def test_set_timestamp_past_14_days_raise_exception(): + context = MetricsContext() + past_date = datetime.datetime.now() - datetime.timedelta(milliseconds=MAX_TIMESTAMP_PAST_AGE_SECONDS + 1) + with pytest.raises(InvalidTimestampError): + context.set_timestamp(past_date) + + +def test_set_timestamp_future_2_hours_raise_exception(): + context = MetricsContext() + future_date = datetime.datetime.now() + datetime.timedelta(milliseconds=MAX_TIMESTAMP_FUTURE_AGE_SECONDS + 1) + + with pytest.raises(InvalidTimestampError): + context.set_timestamp(future_date) From b52f40ba309d8a96611f51cf81e0914ca054bf21 Mon Sep 17 00:00:00 2001 From: Amruth Rayabagi <118546401+rayabagi@users.noreply.github.com> Date: Mon, 11 Sep 2023 11:09:41 -0700 Subject: [PATCH 02/18] Update aws_embedded_metrics/constants.py Co-authored-by: Mark Kuhn --- aws_embedded_metrics/constants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_embedded_metrics/constants.py b/aws_embedded_metrics/constants.py index 590a1fa..45f1b6a 100644 --- a/aws_embedded_metrics/constants.py +++ b/aws_embedded_metrics/constants.py @@ -21,5 +21,5 @@ MAX_NAMESPACE_LENGTH = 256 VALID_NAMESPACE_REGEX = '^[a-zA-Z0-9._#:/-]+$' TIMESTAMP = "Timestamp" -MAX_TIMESTAMP_PAST_AGE_SECONDS = 14 * 24 * 60 * 60 * 1000 # 14 days -MAX_TIMESTAMP_FUTURE_AGE_SECONDS = 2 * 60 * 60 * 1000 # 2 hours +MAX_TIMESTAMP_PAST_AGE = 14 * 24 * 60 * 60 * 1000 # 14 days +MAX_TIMESTAMP_FUTURE_AGE = 2 * 60 * 60 * 1000 # 2 hours From 47fd24a160008aa612146576b0d8883e2ac8774f Mon Sep 17 00:00:00 2001 From: Amruth Rayabagi <118546401+rayabagi@users.noreply.github.com> Date: Mon, 11 Sep 2023 11:24:10 -0700 Subject: [PATCH 03/18] Update aws_embedded_metrics/logger/metrics_context.py Co-authored-by: Mark Kuhn --- aws_embedded_metrics/logger/metrics_context.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/aws_embedded_metrics/logger/metrics_context.py b/aws_embedded_metrics/logger/metrics_context.py index 2aba768..b7a8543 100644 --- a/aws_embedded_metrics/logger/metrics_context.py +++ b/aws_embedded_metrics/logger/metrics_context.py @@ -13,8 +13,7 @@ import datetime -from aws_embedded_metrics import constants, utils -from aws_embedded_metrics import validator +from aws_embedded_metrics import constants, utils, validator from aws_embedded_metrics.config import get_config from aws_embedded_metrics.logger.metric import Metric from aws_embedded_metrics.validator import validate_dimension_set, validate_metric From 3f00ab161cf9454765550d635964d6316e785897 Mon Sep 17 00:00:00 2001 From: Amruth Rayabagi <118546401+rayabagi@users.noreply.github.com> Date: Mon, 11 Sep 2023 11:31:04 -0700 Subject: [PATCH 04/18] Update aws_embedded_metrics/logger/metrics_context.py Co-authored-by: Mark Kuhn --- aws_embedded_metrics/logger/metrics_context.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/aws_embedded_metrics/logger/metrics_context.py b/aws_embedded_metrics/logger/metrics_context.py index b7a8543..9fb12d5 100644 --- a/aws_embedded_metrics/logger/metrics_context.py +++ b/aws_embedded_metrics/logger/metrics_context.py @@ -180,17 +180,14 @@ def empty() -> "MetricsContext": def set_timestamp(self, timestamp: datetime) -> None: """ - Update timestamp field in the metadata - - * @see CloudWatch - * Timestamp + Update the timestamp field in the metadata. Parameters: - timestamp (datetime): timestamp value to be set + timestamp (datetime): The timestamp value to be set. Raises: - InvalidTimestampError: if timestamp is invalid - """ + InvalidTimestampError: If the provided timestamp is invalid. + + """ validator.validate_timestamp(timestamp) self.meta[constants.TIMESTAMP] = utils.convert_to_milliseconds(timestamp) From fd27cab4508c0849b367bf194b079f7cfbec894b Mon Sep 17 00:00:00 2001 From: Amruth Rayabagi <118546401+rayabagi@users.noreply.github.com> Date: Mon, 11 Sep 2023 11:33:08 -0700 Subject: [PATCH 05/18] Update aws_embedded_metrics/validator.py Co-authored-by: Mark Kuhn --- aws_embedded_metrics/validator.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/aws_embedded_metrics/validator.py b/aws_embedded_metrics/validator.py index 8712006..c2bc648 100644 --- a/aws_embedded_metrics/validator.py +++ b/aws_embedded_metrics/validator.py @@ -120,17 +120,15 @@ def validate_namespace(namespace: str) -> None: def validate_timestamp(timestamp: datetime) -> None: """ - Validates a timestamp + Validates a given timestamp based on CloudWatch Timestamp guidelines. - * @see CloudWatch - * Timestamp + For more information, refer to [CloudWatch Timestamp](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#about_timestamp). - Parameters: - timestamp (datetime): datetime to validate + Parameters: + timestamp (datetime): Datetime object representing the timestamp to validate. - Raises: - InvalidTimestampError: if datetime is invalid + Raises: + InvalidTimestampError: If the timestamp is either None, too old, or too far in the future. """ if (timestamp is None): raise InvalidTimestampError(f"Timestamp cannot be none") From 22169da80b6cb2d0a9a82909dc592105fdac8b08 Mon Sep 17 00:00:00 2001 From: Amruth Rayabagi Date: Mon, 11 Sep 2023 15:41:11 -0700 Subject: [PATCH 06/18] Fixed review comments --- .../logger/metrics_context.py | 4 +- aws_embedded_metrics/validator.py | 30 +++++--- tests/logger/test_metrics_context.py | 72 ++++++++++--------- 3 files changed, 61 insertions(+), 45 deletions(-) diff --git a/aws_embedded_metrics/logger/metrics_context.py b/aws_embedded_metrics/logger/metrics_context.py index 9fb12d5..2721c26 100644 --- a/aws_embedded_metrics/logger/metrics_context.py +++ b/aws_embedded_metrics/logger/metrics_context.py @@ -178,10 +178,12 @@ def create_copy_with_context(self, preserve_dimensions: bool = False) -> "Metric def empty() -> "MetricsContext": return MetricsContext() - def set_timestamp(self, timestamp: datetime) -> None: + def set_timestamp(self, timestamp: datetime.datetime) -> None: """ Update the timestamp field in the metadata. + For more information, refer to (https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#about_timestamp). + Parameters: timestamp (datetime): The timestamp value to be set. diff --git a/aws_embedded_metrics/validator.py b/aws_embedded_metrics/validator.py index c2bc648..2228e0e 100644 --- a/aws_embedded_metrics/validator.py +++ b/aws_embedded_metrics/validator.py @@ -16,7 +16,8 @@ from typing import Dict, Optional from aws_embedded_metrics.unit import Unit from aws_embedded_metrics.storage_resolution import StorageResolution -from aws_embedded_metrics.exceptions import DimensionSetExceededError, InvalidDimensionError, InvalidMetricError, InvalidNamespaceError, InvalidTimestampError +from aws_embedded_metrics.exceptions import DimensionSetExceededError, InvalidDimensionError, InvalidMetricError, InvalidNamespaceError +from aws_embedded_metrics.exceptions import InvalidTimestampError import aws_embedded_metrics.constants as constants import datetime from aws_embedded_metrics import utils, constants @@ -122,7 +123,7 @@ def validate_timestamp(timestamp: datetime) -> None: """ Validates a given timestamp based on CloudWatch Timestamp guidelines. - For more information, refer to [CloudWatch Timestamp](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#about_timestamp). + For more information, refer to (https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#about_timestamp). Parameters: timestamp (datetime): Datetime object representing the timestamp to validate. @@ -130,16 +131,23 @@ def validate_timestamp(timestamp: datetime) -> None: Raises: InvalidTimestampError: If the timestamp is either None, too old, or too far in the future. """ - if (timestamp is None): - raise InvalidTimestampError(f"Timestamp cannot be none") + if not timestamp: + raise InvalidTimestampError("Timestamp must be a valid datetime object") + + timestamp_past_age_error_message = f"Timestamp {str(timestamp)} must not be older than {str(constants.MAX_TIMESTAMP_PAST_AGE)} milliseconds" + timestamp_future_age_error_message = f"Timestamp {str(timestamp)} must not be newer than {str(constants.MAX_TIMESTAMP_FUTURE_AGE)} milliseconds" + + if timestamp == datetime.datetime.min: + raise InvalidTimestampError(timestamp_past_age_error_message) + + if timestamp == datetime.datetime.max: + raise InvalidTimestampError(timestamp_future_age_error_message) given_time_in_milliseconds = utils.convert_to_milliseconds(timestamp) - current_time_in_milliseconds = utils.convert_to_milliseconds(datetime.datetime.now()) + current_time_in_milliseconds = utils.now() - if (given_time_in_milliseconds <= (current_time_in_milliseconds - constants.MAX_TIMESTAMP_PAST_AGE_SECONDS)): - raise InvalidTimestampError( - f"Timestamp {str(timestamp)} must not be older than {str(constants.MAX_TIMESTAMP_PAST_AGE_SECONDS)} milliseconds") + if given_time_in_milliseconds < (current_time_in_milliseconds - constants.MAX_TIMESTAMP_PAST_AGE): + raise InvalidTimestampError(timestamp_past_age_error_message) - if (given_time_in_milliseconds >= (current_time_in_milliseconds + constants.MAX_TIMESTAMP_FUTURE_AGE_SECONDS)): - raise InvalidTimestampError( - f"Timestamp {str(timestamp)} must not be newer than {str(constants.MAX_TIMESTAMP_FUTURE_AGE_SECONDS)} milliseconds") + if given_time_in_milliseconds > (current_time_in_milliseconds + constants.MAX_TIMESTAMP_FUTURE_AGE): + raise InvalidTimestampError(timestamp_future_age_error_message) diff --git a/tests/logger/test_metrics_context.py b/tests/logger/test_metrics_context.py index b5327d8..2142407 100644 --- a/tests/logger/test_metrics_context.py +++ b/tests/logger/test_metrics_context.py @@ -1,3 +1,5 @@ +from faker import Faker +from importlib import reload import datetime import pytest import math @@ -7,10 +9,9 @@ from aws_embedded_metrics.storage_resolution import StorageResolution from aws_embedded_metrics import config from aws_embedded_metrics.logger.metrics_context import MetricsContext -from aws_embedded_metrics.constants import DEFAULT_NAMESPACE, MAX_TIMESTAMP_FUTURE_AGE_SECONDS, MAX_TIMESTAMP_PAST_AGE_SECONDS -from aws_embedded_metrics.exceptions import DimensionSetExceededError, InvalidDimensionError, InvalidMetricError, InvalidTimestampError -from importlib import reload -from faker import Faker +from aws_embedded_metrics.constants import DEFAULT_NAMESPACE, MAX_TIMESTAMP_FUTURE_AGE, MAX_TIMESTAMP_PAST_AGE +from aws_embedded_metrics.exceptions import DimensionSetExceededError, InvalidDimensionError, InvalidMetricError +from aws_embedded_metrics.exceptions import InvalidTimestampError fake = Faker() @@ -459,44 +460,49 @@ def test_cannot_put_more_than_30_dimensions(): context.put_dimensions(dimension_set) -# Test utility method - - -def generate_dimension_set(dimensions_to_add): - dimension_set = {} - for i in range(0, dimensions_to_add): - dimension_set[f"{i}"] = fake.word() - - return dimension_set - - -def test_set_timestamp_verify_timestamp(): +@pytest.mark.parametrize( + "timestamp", + [ + datetime.datetime.now(), + datetime.datetime.now() - datetime.timedelta(milliseconds=MAX_TIMESTAMP_PAST_AGE - 5000), + datetime.datetime.now() + datetime.timedelta(milliseconds=MAX_TIMESTAMP_FUTURE_AGE - 5000) + ] +) +def test_set_timestamp_sets_timestamp(timestamp: datetime.datetime): context = MetricsContext() - context.put_metric("TestMetric", 0) - now = datetime.datetime.now() - context.set_timestamp(now) + context.set_timestamp(timestamp) - assert context.meta[constants.TIMESTAMP] == utils.convert_to_milliseconds(now) + assert context.meta[constants.TIMESTAMP] == utils.convert_to_milliseconds(timestamp) -def test_set_timestamp_null_raise_exception(): +@pytest.mark.parametrize( + "timestamp", + [ + None, + datetime.datetime.min, + datetime.datetime.max, + datetime.datetime(1, 1, 1, 0, 0, 0, 0, None), + datetime.datetime(1, 1, 1), + datetime.datetime(1, 1, 1, 0, 0), + datetime.datetime(9999, 12, 31, 23, 59, 59, 999999), + datetime.datetime.now() - datetime.timedelta(milliseconds=MAX_TIMESTAMP_PAST_AGE + 5000), + datetime.datetime.now() + datetime.timedelta(milliseconds=MAX_TIMESTAMP_FUTURE_AGE + 5000) + ] +) +def test_set_timestamp_raise_exception(timestamp: datetime.datetime): context = MetricsContext() - past_date = None + with pytest.raises(InvalidTimestampError): - context.set_timestamp(past_date) + context.set_timestamp(timestamp) -def test_set_timestamp_past_14_days_raise_exception(): - context = MetricsContext() - past_date = datetime.datetime.now() - datetime.timedelta(milliseconds=MAX_TIMESTAMP_PAST_AGE_SECONDS + 1) - with pytest.raises(InvalidTimestampError): - context.set_timestamp(past_date) +# Test utility method -def test_set_timestamp_future_2_hours_raise_exception(): - context = MetricsContext() - future_date = datetime.datetime.now() + datetime.timedelta(milliseconds=MAX_TIMESTAMP_FUTURE_AGE_SECONDS + 1) +def generate_dimension_set(dimensions_to_add): + dimension_set = {} + for i in range(0, dimensions_to_add): + dimension_set[f"{i}"] = fake.word() - with pytest.raises(InvalidTimestampError): - context.set_timestamp(future_date) + return dimension_set From 9e8eb733053361f1e4d8ced4cd56de27b32949c5 Mon Sep 17 00:00:00 2001 From: Amruth Rayabagi Date: Mon, 11 Sep 2023 16:17:38 -0700 Subject: [PATCH 07/18] Updated import statements for datetime --- aws_embedded_metrics/logger/metrics_context.py | 4 ++-- aws_embedded_metrics/validator.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aws_embedded_metrics/logger/metrics_context.py b/aws_embedded_metrics/logger/metrics_context.py index 2721c26..91309ac 100644 --- a/aws_embedded_metrics/logger/metrics_context.py +++ b/aws_embedded_metrics/logger/metrics_context.py @@ -12,7 +12,7 @@ # limitations under the License. -import datetime +from datetime import datetime from aws_embedded_metrics import constants, utils, validator from aws_embedded_metrics.config import get_config from aws_embedded_metrics.logger.metric import Metric @@ -178,7 +178,7 @@ def create_copy_with_context(self, preserve_dimensions: bool = False) -> "Metric def empty() -> "MetricsContext": return MetricsContext() - def set_timestamp(self, timestamp: datetime.datetime) -> None: + def set_timestamp(self, timestamp: datetime) -> None: """ Update the timestamp field in the metadata. diff --git a/aws_embedded_metrics/validator.py b/aws_embedded_metrics/validator.py index 2228e0e..f541f59 100644 --- a/aws_embedded_metrics/validator.py +++ b/aws_embedded_metrics/validator.py @@ -19,7 +19,7 @@ from aws_embedded_metrics.exceptions import DimensionSetExceededError, InvalidDimensionError, InvalidMetricError, InvalidNamespaceError from aws_embedded_metrics.exceptions import InvalidTimestampError import aws_embedded_metrics.constants as constants -import datetime +from datetime import datetime from aws_embedded_metrics import utils, constants @@ -137,10 +137,10 @@ def validate_timestamp(timestamp: datetime) -> None: timestamp_past_age_error_message = f"Timestamp {str(timestamp)} must not be older than {str(constants.MAX_TIMESTAMP_PAST_AGE)} milliseconds" timestamp_future_age_error_message = f"Timestamp {str(timestamp)} must not be newer than {str(constants.MAX_TIMESTAMP_FUTURE_AGE)} milliseconds" - if timestamp == datetime.datetime.min: + if timestamp == datetime.min: raise InvalidTimestampError(timestamp_past_age_error_message) - if timestamp == datetime.datetime.max: + if timestamp == datetime.max: raise InvalidTimestampError(timestamp_future_age_error_message) given_time_in_milliseconds = utils.convert_to_milliseconds(timestamp) From 9f48d9f978b3bb5de27d3266c78916e664a490ea Mon Sep 17 00:00:00 2001 From: Amruth Rayabagi Date: Mon, 11 Sep 2023 17:07:03 -0700 Subject: [PATCH 08/18] import issues resolved --- aws_embedded_metrics/validator.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/aws_embedded_metrics/validator.py b/aws_embedded_metrics/validator.py index f541f59..a167629 100644 --- a/aws_embedded_metrics/validator.py +++ b/aws_embedded_metrics/validator.py @@ -18,9 +18,8 @@ from aws_embedded_metrics.storage_resolution import StorageResolution from aws_embedded_metrics.exceptions import DimensionSetExceededError, InvalidDimensionError, InvalidMetricError, InvalidNamespaceError from aws_embedded_metrics.exceptions import InvalidTimestampError -import aws_embedded_metrics.constants as constants from datetime import datetime -from aws_embedded_metrics import utils, constants +from aws_embedded_metrics import constants, utils def validate_dimension_set(dimension_set: Dict[str, str]) -> None: From 2a0b31986221f83d9bf133dc57646851df5eb980 Mon Sep 17 00:00:00 2001 From: Amruth Rayabagi Date: Fri, 8 Sep 2023 17:32:13 -0700 Subject: [PATCH 09/18] Added support set custom timestamp Co-Authored-By: Mark Kuhn --- aws_embedded_metrics/logger/metrics_context.py | 2 +- aws_embedded_metrics/validator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_embedded_metrics/logger/metrics_context.py b/aws_embedded_metrics/logger/metrics_context.py index 91309ac..72b8241 100644 --- a/aws_embedded_metrics/logger/metrics_context.py +++ b/aws_embedded_metrics/logger/metrics_context.py @@ -182,7 +182,7 @@ def set_timestamp(self, timestamp: datetime) -> None: """ Update the timestamp field in the metadata. - For more information, refer to (https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#about_timestamp). + Timestamp must meet CloudWatch requirements, otherwise a InvalidTimestampError will be thrown. See [Timestamps](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#about_timestamp) for valid values. Parameters: timestamp (datetime): The timestamp value to be set. diff --git a/aws_embedded_metrics/validator.py b/aws_embedded_metrics/validator.py index a167629..353d09c 100644 --- a/aws_embedded_metrics/validator.py +++ b/aws_embedded_metrics/validator.py @@ -122,7 +122,7 @@ def validate_timestamp(timestamp: datetime) -> None: """ Validates a given timestamp based on CloudWatch Timestamp guidelines. - For more information, refer to (https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#about_timestamp). + Timestamp must meet CloudWatch requirements, otherwise a InvalidTimestampError will be thrown. See [Timestamps](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#about_timestamp) for valid values. Parameters: timestamp (datetime): Datetime object representing the timestamp to validate. From f1b89c80591fda02436599864dfc7c59106fbb15 Mon Sep 17 00:00:00 2001 From: Amruth Rayabagi Date: Mon, 11 Sep 2023 17:27:44 -0700 Subject: [PATCH 10/18] Fixed length check issues in build --- aws_embedded_metrics/logger/metrics_context.py | 4 +++- aws_embedded_metrics/validator.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/aws_embedded_metrics/logger/metrics_context.py b/aws_embedded_metrics/logger/metrics_context.py index 72b8241..7a6d105 100644 --- a/aws_embedded_metrics/logger/metrics_context.py +++ b/aws_embedded_metrics/logger/metrics_context.py @@ -182,7 +182,9 @@ def set_timestamp(self, timestamp: datetime) -> None: """ Update the timestamp field in the metadata. - Timestamp must meet CloudWatch requirements, otherwise a InvalidTimestampError will be thrown. See [Timestamps](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#about_timestamp) for valid values. + Timestamp must meet CloudWatch requirements, otherwise a InvalidTimestampError will be thrown. + See [Timestamps](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#about_timestamp) + for valid values. Parameters: timestamp (datetime): The timestamp value to be set. diff --git a/aws_embedded_metrics/validator.py b/aws_embedded_metrics/validator.py index 353d09c..4d7c53a 100644 --- a/aws_embedded_metrics/validator.py +++ b/aws_embedded_metrics/validator.py @@ -122,7 +122,9 @@ def validate_timestamp(timestamp: datetime) -> None: """ Validates a given timestamp based on CloudWatch Timestamp guidelines. - Timestamp must meet CloudWatch requirements, otherwise a InvalidTimestampError will be thrown. See [Timestamps](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#about_timestamp) for valid values. + Timestamp must meet CloudWatch requirements, otherwise a InvalidTimestampError will be thrown. + See [Timestamps](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#about_timestamp) + for valid values. Parameters: timestamp (datetime): Datetime object representing the timestamp to validate. From d17071eb18fa3d70bb747c4070a5ffc075f825b9 Mon Sep 17 00:00:00 2001 From: Amruth Rayabagi Date: Mon, 11 Sep 2023 17:39:37 -0700 Subject: [PATCH 11/18] Code formatting fixed flake8 issues --- aws_embedded_metrics/logger/metrics_context.py | 4 ++-- aws_embedded_metrics/validator.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/aws_embedded_metrics/logger/metrics_context.py b/aws_embedded_metrics/logger/metrics_context.py index 7a6d105..0ff6801 100644 --- a/aws_embedded_metrics/logger/metrics_context.py +++ b/aws_embedded_metrics/logger/metrics_context.py @@ -182,8 +182,8 @@ def set_timestamp(self, timestamp: datetime) -> None: """ Update the timestamp field in the metadata. - Timestamp must meet CloudWatch requirements, otherwise a InvalidTimestampError will be thrown. - See [Timestamps](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#about_timestamp) + Timestamp must meet CloudWatch requirements, otherwise a InvalidTimestampError will be thrown. + See [Timestamps](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#about_timestamp) for valid values. Parameters: diff --git a/aws_embedded_metrics/validator.py b/aws_embedded_metrics/validator.py index 4d7c53a..20512d2 100644 --- a/aws_embedded_metrics/validator.py +++ b/aws_embedded_metrics/validator.py @@ -122,8 +122,8 @@ def validate_timestamp(timestamp: datetime) -> None: """ Validates a given timestamp based on CloudWatch Timestamp guidelines. - Timestamp must meet CloudWatch requirements, otherwise a InvalidTimestampError will be thrown. - See [Timestamps](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#about_timestamp) + Timestamp must meet CloudWatch requirements, otherwise a InvalidTimestampError will be thrown. + See [Timestamps](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#about_timestamp) for valid values. Parameters: From 156504cd81b9e608e8f8f70fe0c805f9822818b9 Mon Sep 17 00:00:00 2001 From: Amruth Rayabagi <118546401+rayabagi@users.noreply.github.com> Date: Tue, 12 Sep 2023 14:52:13 -0700 Subject: [PATCH 12/18] Update aws_embedded_metrics/logger/metrics_context.py Co-authored-by: Mark Kuhn --- aws_embedded_metrics/logger/metrics_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_embedded_metrics/logger/metrics_context.py b/aws_embedded_metrics/logger/metrics_context.py index 0ff6801..ac11ea9 100644 --- a/aws_embedded_metrics/logger/metrics_context.py +++ b/aws_embedded_metrics/logger/metrics_context.py @@ -180,7 +180,7 @@ def empty() -> "MetricsContext": def set_timestamp(self, timestamp: datetime) -> None: """ - Update the timestamp field in the metadata. + Set the timestamp of metrics emitted in this context. If not set, the timestamp will default to the time the context is constructed. Timestamp must meet CloudWatch requirements, otherwise a InvalidTimestampError will be thrown. See [Timestamps](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#about_timestamp) From 1c0e7a58b59414734b5f81a7aa9cb4844abebd8b Mon Sep 17 00:00:00 2001 From: Amruth Rayabagi <118546401+rayabagi@users.noreply.github.com> Date: Tue, 12 Sep 2023 14:53:05 -0700 Subject: [PATCH 13/18] Update tests/logger/test_metrics_context.py Co-authored-by: Mark Kuhn --- tests/logger/test_metrics_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/logger/test_metrics_context.py b/tests/logger/test_metrics_context.py index 2142407..e04b7d2 100644 --- a/tests/logger/test_metrics_context.py +++ b/tests/logger/test_metrics_context.py @@ -490,7 +490,7 @@ def test_set_timestamp_sets_timestamp(timestamp: datetime.datetime): datetime.datetime.now() + datetime.timedelta(milliseconds=MAX_TIMESTAMP_FUTURE_AGE + 5000) ] ) -def test_set_timestamp_raise_exception(timestamp: datetime.datetime): +def test_set_invalid_timestamp_raises_exception(timestamp: datetime): context = MetricsContext() with pytest.raises(InvalidTimestampError): From a930402c06d3928673c1b525f00ff9d2b6ceb521 Mon Sep 17 00:00:00 2001 From: Amruth Rayabagi <118546401+rayabagi@users.noreply.github.com> Date: Tue, 12 Sep 2023 14:53:25 -0700 Subject: [PATCH 14/18] Update tests/logger/test_metrics_context.py Co-authored-by: Mark Kuhn --- tests/logger/test_metrics_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/logger/test_metrics_context.py b/tests/logger/test_metrics_context.py index e04b7d2..78539ff 100644 --- a/tests/logger/test_metrics_context.py +++ b/tests/logger/test_metrics_context.py @@ -1,6 +1,6 @@ from faker import Faker from importlib import reload -import datetime +from datetime import datetime import pytest import math import random From 957b1c12f5d8322d33df9f4206245da136788108 Mon Sep 17 00:00:00 2001 From: Amruth Rayabagi Date: Tue, 12 Sep 2023 15:05:46 -0700 Subject: [PATCH 15/18] Added set_timestamp method in metric logger --- aws_embedded_metrics/logger/metrics_logger.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/aws_embedded_metrics/logger/metrics_logger.py b/aws_embedded_metrics/logger/metrics_logger.py index c25cae2..ebbe469 100644 --- a/aws_embedded_metrics/logger/metrics_logger.py +++ b/aws_embedded_metrics/logger/metrics_logger.py @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from datetime import datetime from aws_embedded_metrics.environment import Environment from aws_embedded_metrics.logger.metrics_context import MetricsContext from aws_embedded_metrics.validator import validate_namespace @@ -114,6 +115,10 @@ def add_stack_trace(self, key: str, details: Any = None, exc_info: Tuple = None) self.set_property(key, trace_value) return self + def set_timestamp(self, timestamp: datetime) -> "MetricsLogger": + self.context.set_timestamp(timestamp) + return self + def new(self) -> "MetricsLogger": return MetricsLogger( self.resolve_environment, self.context.create_copy_with_context() From d6a5a273143e3901a9e96b5a50c6ba8a95e00661 Mon Sep 17 00:00:00 2001 From: Amruth Rayabagi Date: Tue, 12 Sep 2023 16:56:38 -0700 Subject: [PATCH 16/18] Fixed review comments --- aws_embedded_metrics/utils.py | 3 +++ aws_embedded_metrics/validator.py | 15 ++++----------- tests/integ/agent/test_end_to_end.py | 1 + tests/logger/test_metrics_context.py | 27 ++++++++++++++------------- tests/logger/test_metrics_logger.py | 18 +++++++++++++++++- 5 files changed, 39 insertions(+), 25 deletions(-) diff --git a/aws_embedded_metrics/utils.py b/aws_embedded_metrics/utils.py index a76c2a7..50ffbda 100644 --- a/aws_embedded_metrics/utils.py +++ b/aws_embedded_metrics/utils.py @@ -17,4 +17,7 @@ def now() -> int: return int(round(time.time() * 1000)) def convert_to_milliseconds(datetime: datetime) -> int: + if datetime == datetime.min: + return 0 + return int(round(datetime.timestamp() * 1000)) diff --git a/aws_embedded_metrics/validator.py b/aws_embedded_metrics/validator.py index 20512d2..d6ac3fe 100644 --- a/aws_embedded_metrics/validator.py +++ b/aws_embedded_metrics/validator.py @@ -135,20 +135,13 @@ def validate_timestamp(timestamp: datetime) -> None: if not timestamp: raise InvalidTimestampError("Timestamp must be a valid datetime object") - timestamp_past_age_error_message = f"Timestamp {str(timestamp)} must not be older than {str(constants.MAX_TIMESTAMP_PAST_AGE)} milliseconds" - timestamp_future_age_error_message = f"Timestamp {str(timestamp)} must not be newer than {str(constants.MAX_TIMESTAMP_FUTURE_AGE)} milliseconds" - - if timestamp == datetime.min: - raise InvalidTimestampError(timestamp_past_age_error_message) - - if timestamp == datetime.max: - raise InvalidTimestampError(timestamp_future_age_error_message) - given_time_in_milliseconds = utils.convert_to_milliseconds(timestamp) current_time_in_milliseconds = utils.now() if given_time_in_milliseconds < (current_time_in_milliseconds - constants.MAX_TIMESTAMP_PAST_AGE): - raise InvalidTimestampError(timestamp_past_age_error_message) + raise InvalidTimestampError( + f"Timestamp {str(timestamp)} must not be older than {int(constants.MAX_TIMESTAMP_PAST_AGE/(24 * 60 * 60 * 1000))} days") if given_time_in_milliseconds > (current_time_in_milliseconds + constants.MAX_TIMESTAMP_FUTURE_AGE): - raise InvalidTimestampError(timestamp_future_age_error_message) + raise InvalidTimestampError( + f"Timestamp {str(timestamp)} must not be newer than {int(constants.MAX_TIMESTAMP_FUTURE_AGE/(60 * 60 * 1000))} hours") diff --git a/tests/integ/agent/test_end_to_end.py b/tests/integ/agent/test_end_to_end.py index 9bb363e..8b45d39 100644 --- a/tests/integ/agent/test_end_to_end.py +++ b/tests/integ/agent/test_end_to_end.py @@ -42,6 +42,7 @@ async def do_work(metrics): metrics.put_dimensions({"Operation": "Agent"}) metrics.put_metric(metric_name, 100, "Milliseconds") metrics.set_property("RequestId", "422b1569-16f6-4a03-b8f0-fe3fd9b100f8") + metrics.set_timestamp(datetime.utcnow()) # act await do_work() diff --git a/tests/logger/test_metrics_context.py b/tests/logger/test_metrics_context.py index 78539ff..a0ab133 100644 --- a/tests/logger/test_metrics_context.py +++ b/tests/logger/test_metrics_context.py @@ -1,6 +1,6 @@ from faker import Faker from importlib import reload -from datetime import datetime +from datetime import datetime, timedelta import pytest import math import random @@ -463,12 +463,12 @@ def test_cannot_put_more_than_30_dimensions(): @pytest.mark.parametrize( "timestamp", [ - datetime.datetime.now(), - datetime.datetime.now() - datetime.timedelta(milliseconds=MAX_TIMESTAMP_PAST_AGE - 5000), - datetime.datetime.now() + datetime.timedelta(milliseconds=MAX_TIMESTAMP_FUTURE_AGE - 5000) + datetime.now(), + datetime.now() - timedelta(milliseconds=MAX_TIMESTAMP_PAST_AGE - 5000), + datetime.now() + timedelta(milliseconds=MAX_TIMESTAMP_FUTURE_AGE - 5000) ] ) -def test_set_timestamp_sets_timestamp(timestamp: datetime.datetime): +def test_set_valid_timestamp_verify_timestamp(timestamp: datetime): context = MetricsContext() context.set_timestamp(timestamp) @@ -480,14 +480,15 @@ def test_set_timestamp_sets_timestamp(timestamp: datetime.datetime): "timestamp", [ None, - datetime.datetime.min, - datetime.datetime.max, - datetime.datetime(1, 1, 1, 0, 0, 0, 0, None), - datetime.datetime(1, 1, 1), - datetime.datetime(1, 1, 1, 0, 0), - datetime.datetime(9999, 12, 31, 23, 59, 59, 999999), - datetime.datetime.now() - datetime.timedelta(milliseconds=MAX_TIMESTAMP_PAST_AGE + 5000), - datetime.datetime.now() + datetime.timedelta(milliseconds=MAX_TIMESTAMP_FUTURE_AGE + 5000) + datetime.min, + datetime(1970, 1, 1, 0, 0, 0), + datetime.max, + datetime(9999, 12, 31, 23, 59, 59, 999999), + datetime(1, 1, 1, 0, 0, 0, 0, None), + datetime(1, 1, 1), + datetime(1, 1, 1, 0, 0), + datetime.now() - timedelta(milliseconds=MAX_TIMESTAMP_PAST_AGE + 1), + datetime.now() + timedelta(milliseconds=MAX_TIMESTAMP_FUTURE_AGE + 5000) ] ) def test_set_invalid_timestamp_raises_exception(timestamp: datetime): diff --git a/tests/logger/test_metrics_logger.py b/tests/logger/test_metrics_logger.py index 8602c47..08bd971 100644 --- a/tests/logger/test_metrics_logger.py +++ b/tests/logger/test_metrics_logger.py @@ -1,4 +1,5 @@ -from aws_embedded_metrics import config +from datetime import datetime +from aws_embedded_metrics import config, utils from aws_embedded_metrics.logger import metrics_logger from aws_embedded_metrics.sinks import Sink from aws_embedded_metrics.environment import Environment @@ -493,6 +494,21 @@ async def test_configure_flush_to_preserve_dimensions(mocker): assert dimensions[0][dimension_key] == dimension_value +@pytest.mark.asyncio +async def test_can_set_timestamp(mocker): + # arrange + expected_value = datetime.now() + + logger, sink, env = get_logger_and_sink(mocker) + + # act + logger.set_timestamp(expected_value) + await logger.flush() + + # assert + context = get_flushed_context(sink) + assert context.meta[constants.TIMESTAMP] == utils.convert_to_milliseconds(expected_value) + # Test helper methods From 96b4aec220dc17ccb889662938025430f140c736 Mon Sep 17 00:00:00 2001 From: Amruth Rayabagi <118546401+rayabagi@users.noreply.github.com> Date: Wed, 13 Sep 2023 09:52:44 -0700 Subject: [PATCH 17/18] Update aws_embedded_metrics/utils.py Co-authored-by: Mark Kuhn --- aws_embedded_metrics/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aws_embedded_metrics/utils.py b/aws_embedded_metrics/utils.py index 50ffbda..8753cd6 100644 --- a/aws_embedded_metrics/utils.py +++ b/aws_embedded_metrics/utils.py @@ -16,8 +16,8 @@ def now() -> int: return int(round(time.time() * 1000)) -def convert_to_milliseconds(datetime: datetime) -> int: - if datetime == datetime.min: +def convert_to_milliseconds(dt: datetime) -> int: + if dt == datetime.min: return 0 - return int(round(datetime.timestamp() * 1000)) + return int(round(dt.timestamp() * 1000)) From 630d488d55190c061ceb951aa48463d57ec2cad8 Mon Sep 17 00:00:00 2001 From: Amruth Rayabagi Date: Wed, 13 Sep 2023 14:29:30 -0700 Subject: [PATCH 18/18] Bumped version to 3.2.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 43e81e9..27c3417 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="aws-embedded-metrics", - version="3.1.1", + version="3.2.0", author="Amazon Web Services", author_email="jarnance@amazon.com", description="AWS Embedded Metrics Package",