localstack · joe4dev · Oct 27, 2023 · Oct 25, 2023 · Oct 25, 2023 · Oct 25, 2023
diff --git a/localstack/deprecations.py b/localstack/deprecations.py
@@ -307,6 +307,7 @@ def log_deprecation_warnings(deprecations: Optional[List[EnvVarDeprecation]] = N
     if provider_override_lambda and provider_override_lambda in ["v1", "legacy"]:
         env_var_value = f"PROVIDER_OVERRIDE_LAMBDA={provider_override_lambda}"
         deprecation_version = "2.0.0"
+        # TODO[LambdaV1] adjust message or convert into generic deprecation for PROVIDER_OVERRIDE_LAMBDA
         deprecation_path = (
             f"Remove {env_var_value} to use the new Lambda 'v2' provider (current default). "
             "For more details, refer to our Lambda migration guide "

diff --git a/localstack/services/cloudformation/models/iam.py b/localstack/services/cloudformation/models/iam.py
@@ -16,13 +16,14 @@
 )
 from localstack.services.cloudformation.service_models import GenericBaseModel
 from localstack.services.iam.provider import SERVICE_LINKED_ROLE_PATH_PREFIX
-from localstack.services.lambda_.lambda_api import IAM_POLICY_VERSION
 from localstack.utils.aws import arns
 from localstack.utils.common import ensure_list
 from localstack.utils.functions import call_safe
 
 LOG = logging.getLogger(__name__)
 
+DEFAULT_IAM_POLICY_VERSION = "2012-10-17"
+
 
 class IAMManagedPolicy(GenericBaseModel):
     @staticmethod
@@ -363,7 +364,7 @@ def _post_create(
             doc = dict(policy["PolicyDocument"])
             doc = remove_none_values(doc)
 
-            doc["Version"] = doc.get("Version") or IAM_POLICY_VERSION
+            doc["Version"] = doc.get("Version") or DEFAULT_IAM_POLICY_VERSION
             statements = ensure_list(doc["Statement"])
             for statement in statements:
                 if isinstance(statement.get("Resource"), list):

diff --git a/localstack/services/lambda_/api_utils.py b/localstack/services/lambda_/api_utils.py
@@ -1,4 +1,6 @@
-""" Utilities for the new Lambda ASF provider. Do not use in the current provider, as ASF specific exceptions might be thrown """
+""" Utilities related to Lambda API operations such as ARN handling, validations, and output formatting.
+Everything related to behavior or implicit functionality goes into `lambda_utils.py`.
+"""
 import datetime
 import random
 import re

diff --git a/localstack/services/lambda_/event_source_listeners/adapters.py b/localstack/services/lambda_/event_source_listeners/adapters.py
@@ -15,10 +15,10 @@
 from localstack.services.lambda_.invocation.lambda_models import InvocationResult
 from localstack.services.lambda_.invocation.lambda_service import LambdaService
 from localstack.services.lambda_.invocation.models import lambda_stores
-from localstack.services.lambda_.lambda_executors import (
+from localstack.services.lambda_.legacy.lambda_executors import (
     InvocationResult as LegacyInvocationResult,  # TODO: extract
 )
-from localstack.services.lambda_.lambda_utils import event_source_arn_matches
+from localstack.services.lambda_.legacy.lambda_utils import event_source_arn_matches
 from localstack.utils.aws.client_types import ServicePrincipal
 from localstack.utils.json import BytesEncoder
 from localstack.utils.strings import to_bytes, to_str
@@ -71,7 +71,7 @@ def __init__(self):
         pass
 
     def invoke(self, function_arn, context, payload, invocation_type, callback=None):
-        from localstack.services.lambda_.lambda_api import run_lambda
+        from localstack.services.lambda_.legacy.lambda_api import run_lambda
 
         try:
             json.dumps(payload)
@@ -97,8 +97,8 @@ def invoke_with_statuscode(
         lock_discriminator,
         parallelization_factor
     ) -> int:
-        from localstack.services.lambda_ import lambda_executors
-        from localstack.services.lambda_.lambda_api import run_lambda
+        from localstack.services.lambda_.legacy import lambda_executors
+        from localstack.services.lambda_.legacy.lambda_api import run_lambda
 
         if not config.SYNCHRONOUS_KINESIS_EVENTS:
             lambda_executors.LAMBDA_ASYNC_LOCKS.assure_lock_present(
@@ -124,7 +124,7 @@ def invoke_with_statuscode(
         return status_code
 
     def get_event_sources(self, source_arn: str) -> list:
-        from localstack.services.lambda_.lambda_api import get_event_sources
+        from localstack.services.lambda_.legacy.lambda_api import get_event_sources
 
         return get_event_sources(source_arn=source_arn)
 

diff --git a/localstack/services/lambda_/event_source_listeners/sqs_event_source_listener.py b/localstack/services/lambda_/event_source_listeners/sqs_event_source_listener.py
@@ -11,11 +11,11 @@
 from localstack.services.lambda_.event_source_listeners.event_source_listener import (
     EventSourceListener,
 )
-from localstack.services.lambda_.lambda_executors import InvocationResult
-from localstack.services.lambda_.lambda_utils import (
+from localstack.services.lambda_.event_source_listeners.utils import (
     filter_stream_records,
     message_attributes_to_lower,
 )
+from localstack.services.lambda_.legacy.lambda_executors import InvocationResult
 from localstack.utils.aws import arns
 from localstack.utils.aws.arns import extract_region_from_arn
 from localstack.utils.threads import FuncThread

diff --git a/localstack/services/lambda_/event_source_listeners/stream_event_source_listener.py b/localstack/services/lambda_/event_source_listeners/stream_event_source_listener.py
@@ -13,7 +13,9 @@
 from localstack.services.lambda_.event_source_listeners.event_source_listener import (
     EventSourceListener,
 )
-from localstack.services.lambda_.lambda_utils import filter_stream_records
+from localstack.services.lambda_.event_source_listeners.utils import (
+    filter_stream_records,
+)
 from localstack.utils.aws.arns import extract_region_from_arn
 from localstack.utils.aws.message_forwarding import send_event_to_target
 from localstack.utils.common import long_uid, timestamp_millis

diff --git a/localstack/services/lambda_/event_source_listeners/utils.py b/localstack/services/lambda_/event_source_listeners/utils.py
@@ -0,0 +1,138 @@
+import json
+import logging
+from typing import Dict, List, Union
+
+from localstack.aws.api.lambda_ import FilterCriteria
+from localstack.utils.strings import first_char_to_lower
+
+LOG = logging.getLogger(__name__)
+
+
+def filter_stream_records(records, filters: List[FilterCriteria]):
+    filtered_records = []
+    for record in records:
+        for filter in filters:
+            for rule in filter["Filters"]:
+                if filter_stream_record(json.loads(rule["Pattern"]), record):
+                    filtered_records.append(record)
+                    break
+    return filtered_records
+
+
+def filter_stream_record(filter_rule: Dict[str, any], record: Dict[str, any]) -> bool:
+    if not filter_rule:
+        return True
+    # https://docs.aws.amazon.com/lambda/latest/dg/invocation-eventfiltering.html#filtering-syntax
+    filter_results = []
+    for key, value in filter_rule.items():
+        # check if rule exists in event
+        record_value = (
+            record.get(key.lower(), record.get(key)) if isinstance(record, Dict) else None
+        )
+        append_record = False
+        if record_value is not None:
+            # check if filter rule value is a list (leaf of rule tree) or a dict (rescursively call function)
+            if isinstance(value, list):
+                if len(value) > 0:
+                    if isinstance(value[0], (str, int)):
+                        append_record = record_value in value
+                    if isinstance(value[0], dict):
+                        append_record = verify_dict_filter(record_value, value[0])
+                else:
+                    LOG.warning(f"Empty lambda filter: {key}")
+            elif isinstance(value, dict):
+                append_record = filter_stream_record(value, record_value)
+        else:
+            # special case 'exists'
+            if isinstance(value, list) and len(value) > 0:
+                append_record = not value[0].get("exists", True)
+
+        filter_results.append(append_record)
+    return all(filter_results)
+
+
+def verify_dict_filter(record_value: any, dict_filter: Dict[str, any]) -> bool:
+    # https://docs.aws.amazon.com/lambda/latest/dg/invocation-eventfiltering.html#filtering-syntax
+    fits_filter = False
+    for key, filter_value in dict_filter.items():
+        if key.lower() == "anything-but":
+            fits_filter = record_value not in filter_value
+        elif key.lower() == "numeric":
+            fits_filter = parse_and_apply_numeric_filter(record_value, filter_value)
+        elif key.lower() == "exists":
+            fits_filter = bool(filter_value)  # exists means that the key exists in the event record
+        elif key.lower() == "prefix":
+            if not isinstance(record_value, str):
+                LOG.warning(f"Record Value {record_value} does not seem to be a valid string.")
+            fits_filter = isinstance(record_value, str) and record_value.startswith(
+                str(filter_value)
+            )
+
+        if fits_filter:
+            return True
+    return fits_filter
+
+
+def parse_and_apply_numeric_filter(
+    record_value: Dict, numeric_filter: List[Union[str, int]]
+) -> bool:
+    if len(numeric_filter) % 2 > 0:
+        LOG.warning("Invalid numeric lambda filter given")
+        return True
+
+    if not isinstance(record_value, (int, float)):
+        LOG.warning(f"Record {record_value} seem not to be a valid number")
+        return False
+
+    for idx in range(0, len(numeric_filter), 2):
+        try:
+            if numeric_filter[idx] == ">" and not (record_value > float(numeric_filter[idx + 1])):
+                return False
+            if numeric_filter[idx] == ">=" and not (record_value >= float(numeric_filter[idx + 1])):
+                return False
+            if numeric_filter[idx] == "=" and not (record_value == float(numeric_filter[idx + 1])):
+                return False
+            if numeric_filter[idx] == "<" and not (record_value < float(numeric_filter[idx + 1])):
+                return False
+            if numeric_filter[idx] == "<=" and not (record_value <= float(numeric_filter[idx + 1])):
+                return False
+        except ValueError:
+            LOG.warning(
+                f"Could not convert filter value {numeric_filter[idx + 1]} to a valid number value for filtering"
+            )
+    return True
+
+
+def contains_list(filter: Dict) -> bool:
+    if isinstance(filter, dict):
+        for key, value in filter.items():
+            if isinstance(value, list) and len(value) > 0:
+                return True
+            return contains_list(value)
+    return False
+
+
+def validate_filters(filter: FilterCriteria) -> bool:
+    # filter needs to be json serializeable
+    for rule in filter["Filters"]:
+        try:
+            if not (filter_pattern := json.loads(rule["Pattern"])):
+                return False
+            return contains_list(filter_pattern)
+        except json.JSONDecodeError:
+            return False
+    # needs to contain on what to filter (some list with citerias)
+    # https://docs.aws.amazon.com/lambda/latest/dg/invocation-eventfiltering.html#filtering-syntax
+
+    return True
+
+
+def message_attributes_to_lower(message_attrs):
+    """Convert message attribute details (first characters) to lower case (e.g., stringValue, dataType)."""
+    message_attrs = message_attrs or {}
+    for _, attr in message_attrs.items():
+        if not isinstance(attr, dict):
+            continue
+        for key, value in dict(attr).items():
+            attr[first_char_to_lower(key)] = attr.pop(key)
+    return message_attrs
diff --git a/localstack/services/lambda_/hooks.py b/localstack/services/lambda_/hooks.py
@@ -1,3 +1,4 @@
+"""Definition of Plux extension points (i.e., hooks) for Lambda."""
 from localstack.runtime.hooks import hook_spec
 
 HOOKS_LAMBDA_START_DOCKER_EXECUTOR = "localstack.hooks.lambda_start_docker_executor"

diff --git a/localstack/services/lambda_/invocation/docker_runtime_executor.py b/localstack/services/lambda_/invocation/docker_runtime_executor.py
@@ -18,8 +18,8 @@
     LambdaRuntimeException,
     RuntimeExecutor,
 )
-from localstack.services.lambda_.lambda_utils import (
-    HINT_LOG,
+from localstack.services.lambda_.lambda_utils import HINT_LOG
+from localstack.services.lambda_.networking import (
     get_all_container_networks_for_lambda,
     get_main_endpoint_from_container,
 )

diff --git a/localstack/services/lambda_/invocation/event_manager.py b/localstack/services/lambda_/invocation/event_manager.py
@@ -18,7 +18,7 @@
     InvocationResult,
 )
 from localstack.services.lambda_.invocation.version_manager import LambdaVersionManager
-from localstack.services.lambda_.lambda_executors import InvocationException
+from localstack.services.lambda_.legacy.lambda_executors import InvocationException
 from localstack.utils.aws import dead_letter_queue
 from localstack.utils.aws.message_forwarding import send_event_to_target
 from localstack.utils.strings import md5, to_str

diff --git a/localstack/services/lambda_/invocation/lambda_models.py b/localstack/services/lambda_/invocation/lambda_models.py
@@ -1,3 +1,7 @@
+"""Lambda models for internal use and persistence.
+The LambdaProviderPro in localstack-ext imports this model and configures persistence.
+The actual function code is stored in S3 (see S3Code).
+"""
 import dataclasses
 import logging
 import shutil