From 97690223b10a95dd8309adaf68bf048496d9205e Mon Sep 17 00:00:00 2001 From: Misha Tiurin Date: Sun, 25 May 2025 14:49:24 +0200 Subject: [PATCH 1/2] Add init parameter for shards When updating a pipe, its new poller needs to pick up where the old one left off. --- .../lambda_/event_source_mapping/pollers/dynamodb_poller.py | 2 ++ .../lambda_/event_source_mapping/pollers/kinesis_poller.py | 2 ++ .../lambda_/event_source_mapping/pollers/stream_poller.py | 3 ++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/dynamodb_poller.py b/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/dynamodb_poller.py index 2a8e793945c42..9733d208d3fc6 100644 --- a/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/dynamodb_poller.py +++ b/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/dynamodb_poller.py @@ -21,6 +21,7 @@ def __init__( processor: EventProcessor | None = None, partner_resource_arn: str | None = None, esm_uuid: str | None = None, + shards: dict[str, str] | None = None, ): super().__init__( source_arn, @@ -29,6 +30,7 @@ def __init__( processor, esm_uuid=esm_uuid, partner_resource_arn=partner_resource_arn, + shards=shards, ) @property diff --git a/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/kinesis_poller.py b/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/kinesis_poller.py index e2dc19b74b012..defe87a6a6dee 100644 --- a/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/kinesis_poller.py +++ b/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/kinesis_poller.py @@ -37,6 +37,7 @@ def __init__( invoke_identity_arn: str | None = None, kinesis_namespace: bool = False, esm_uuid: str | None = None, + shards: dict[str, str] | None = None, ): super().__init__( source_arn, @@ -45,6 +46,7 @@ def __init__( processor, esm_uuid=esm_uuid, partner_resource_arn=partner_resource_arn, + shards=shards, ) self.invoke_identity_arn = invoke_identity_arn self.kinesis_namespace = kinesis_namespace diff --git a/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/stream_poller.py b/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/stream_poller.py index 72d7c3ef3523b..33e95eb32756b 100644 --- a/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/stream_poller.py +++ b/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/stream_poller.py @@ -70,11 +70,12 @@ def __init__( processor: EventProcessor | None = None, partner_resource_arn: str | None = None, esm_uuid: str | None = None, + shards: dict[str, str] | None = None, ): super().__init__(source_arn, source_parameters, source_client, processor) self.partner_resource_arn = partner_resource_arn self.esm_uuid = esm_uuid - self.shards = {} + self.shards = shards if shards is not None else {} self.iterator_over_shards = None self._is_shutdown = threading.Event() From 38620e3ef995104cca416edf324dc607891bd9a2 Mon Sep 17 00:00:00 2001 From: Misha Tiurin Date: Mon, 26 May 2025 01:17:52 +0200 Subject: [PATCH 2/2] Use same util function to get current time datetime.utcnow() is deprecated and was giving a 2-hour difference result than get_current_time() for a computer running in CEST timezone. This led to valid records being expired in bisect_events_by_record_age in stream poller. Same function needs to be used across the code, especially when a comparison is made within same logic. --- .../lambda_/event_source_mapping/pollers/dynamodb_poller.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/dynamodb_poller.py b/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/dynamodb_poller.py index 9733d208d3fc6..d8b1af71b1b71 100644 --- a/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/dynamodb_poller.py +++ b/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/dynamodb_poller.py @@ -7,6 +7,7 @@ from localstack.services.lambda_.event_source_mapping.event_processor import ( EventProcessor, ) +from localstack.services.lambda_.event_source_mapping.pipe_utils import get_current_time from localstack.services.lambda_.event_source_mapping.pollers.stream_poller import StreamPoller LOG = logging.getLogger(__name__) @@ -109,7 +110,7 @@ def get_approximate_arrival_time(self, record: dict) -> float: # Optional according to AWS docs: # https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_streams_StreamRecord.html # TODO: parse float properly if present from ApproximateCreationDateTime -> now works, compare via debug! - return record["dynamodb"].get("todo", datetime.utcnow().timestamp()) + return record["dynamodb"].get("todo", get_current_time().timestamp()) def format_datetime(self, time: datetime) -> str: return f"{time.isoformat(timespec='seconds')}Z"