From c39f8a557ca551ebc2dc22a7157f69281008f612 Mon Sep 17 00:00:00 2001 From: Daniel Fangl Date: Wed, 30 Nov 2022 16:46:49 +0100 Subject: [PATCH 1/5] first implementation of container image crud / implement hooks for pro features --- localstack/services/awslambda/api_utils.py | 16 +++- .../invocation/docker_runtime_executor.py | 73 +++++++++++++------ .../awslambda/invocation/lambda_models.py | 1 + localstack/services/awslambda/provider.py | 49 ++++++++++--- .../utils/container_utils/container_client.py | 11 +++ .../container_utils/docker_cmd_client.py | 13 ++++ .../container_utils/docker_sdk_client.py | 7 ++ 7 files changed, 135 insertions(+), 35 deletions(-) diff --git a/localstack/services/awslambda/api_utils.py b/localstack/services/awslambda/api_utils.py index a82b297c2c81f..9e12b8b800dd1 100644 --- a/localstack/services/awslambda/api_utils.py +++ b/localstack/services/awslambda/api_utils.py @@ -374,6 +374,20 @@ def map_config_out( {"Arn": layer.layer_version_arn, "CodeSize": layer.code.code_size} for layer in version.config.layers ] + if version.config.image_config: + image_config_response = {} + if version.config.image_config.command: + image_config_response["Command"] = version.config.image_config.command + if version.config.image_config.entrypoint: + image_config_response["EntryPoint"] = version.config.image_config.entrypoint + if version.config.image_config.working_directory: + image_config_response[ + "WorkingDirectory" + ] = version.config.image_config.working_directory + optional_kwargs["ImageConfigResponse"] = image_config_response + if version.config.code: + optional_kwargs["CodeSize"] = version.config.code.code_size + optional_kwargs["CodeSha256"] = version.config.code.code_sha256 func_conf = FunctionConfiguration( RevisionId=version.config.revision_id, @@ -388,8 +402,6 @@ def map_config_out( Timeout=version.config.timeout, Runtime=version.config.runtime, Handler=version.config.handler, - CodeSize=version.config.code.code_size, - CodeSha256=version.config.code.code_sha256, MemorySize=version.config.memory_size, PackageType=version.config.package_type, TracingConfig=TracingConfig(Mode=version.config.tracing_config_mode), diff --git a/localstack/services/awslambda/invocation/docker_runtime_executor.py b/localstack/services/awslambda/invocation/docker_runtime_executor.py index 6592d14aeb517..8db7bbf086f84 100644 --- a/localstack/services/awslambda/invocation/docker_runtime_executor.py +++ b/localstack/services/awslambda/invocation/docker_runtime_executor.py @@ -1,3 +1,4 @@ +import dataclasses import json import logging import shutil @@ -6,6 +7,8 @@ from typing import Dict, Literal, Optional from localstack import config +from localstack.aws.api.lambda_ import PackageType +from localstack.runtime.hooks import hook_spec from localstack.services.awslambda.invocation.executor_endpoint import ( ExecutorEndpoint, ServiceEndpoint, @@ -24,6 +27,13 @@ from localstack.utils.docker_utils import DOCKER_CLIENT as CONTAINER_CLIENT from localstack.utils.strings import truncate +# Hook definitions +HOOKS_LAMBDA_START_DOCKER_EXECUTOR = "localstack.hooks.lambda_start_docker_executor" +HOOKS_LAMBDA_PREPARE_DOCKER_EXECUTOR = "localstack.hooks.lambda_prepare_docker_executors" + +start_docker_executor = hook_spec(HOOKS_LAMBDA_START_DOCKER_EXECUTOR) +prepare_docker_executor = hook_spec(HOOKS_LAMBDA_PREPARE_DOCKER_EXECUTOR) + LOG = logging.getLogger(__name__) RUNTIME_REGEX = r"(?P[a-z]+)(?P\d+(\.\d+)?(\.al2)?)(?:.*)" @@ -96,6 +106,11 @@ def prepare_image(target_path: Path, function_version: FunctionVersion) -> None: ) +@dataclasses.dataclass +class LambdaContainerConfiguration(ContainerConfiguration): + copy_folders: list[tuple[str, str]] = dataclasses.field(default_factory=list) + + class DockerRuntimeExecutor(RuntimeExecutor): ip: Optional[str] executor_endpoint: Optional[ExecutorEndpoint] @@ -135,23 +150,35 @@ def _build_executor_endpoint(self, service_endpoint: ServiceEndpoint) -> Executo def start(self, env_vars: dict[str, str]) -> None: self.executor_endpoint.start() network = self._get_network_for_executor() - container_config = ContainerConfiguration( - image_name=self.get_image(), + container_config = LambdaContainerConfiguration( + image_name=None, name=self.id, env_vars=env_vars, network=network, entrypoint=RAPID_ENTRYPOINT, ) + start_docker_executor.run(container_config, self.function_version) + + if not container_config.image_name: + container_config.image_name = self.get_image() + CONTAINER_CLIENT.create_container_from_config(container_config) - if not config.LAMBDA_PREBUILD_IMAGES: + if ( + not config.LAMBDA_PREBUILD_IMAGES + or self.function_version.config.package_type != PackageType.Zip + ): CONTAINER_CLIENT.copy_into_container( self.id, str(get_runtime_client_path()), RAPID_ENTRYPOINT ) - CONTAINER_CLIENT.copy_into_container( - self.id, - f"{str(self.function_version.config.code.get_unzipped_code_location())}/.", - "/var/task", - ) + if self.function_version.config.package_type == PackageType.Zip: + if not config.LAMBDA_PREBUILD_IMAGES: + CONTAINER_CLIENT.copy_into_container( + self.id, + f"{str(self.function_version.config.code.get_unzipped_code_location())}/.", + "/var/task", + ) + for source, target in container_config.copy_folders: + CONTAINER_CLIENT.copy_into_container(self.id, source, target) CONTAINER_CLIENT.start_container(self.id) self.ip = CONTAINER_CLIENT.get_container_ipv4_for_network( @@ -193,19 +220,23 @@ def invoke(self, payload: Dict[str, str]): @classmethod def prepare_version(cls, function_version: FunctionVersion) -> None: time_before = time.perf_counter() - function_version.config.code.prepare_for_execution() - target_path = function_version.config.code.get_unzipped_code_location() - image_name = get_image_for_runtime(function_version.config.runtime) - if image_name not in PULLED_IMAGES: - CONTAINER_CLIENT.pull_image(image_name) - PULLED_IMAGES.add(image_name) - if config.LAMBDA_PREBUILD_IMAGES: - prepare_image(target_path, function_version) - LOG.debug( - "Version preparation of version %s took %0.2fms", - function_version.qualified_arn, - (time.perf_counter() - time_before) * 1000, - ) + prepare_docker_executor.run(function_version) + if function_version.config.code: + function_version.config.code.prepare_for_execution() + for layer in function_version.config.layers: + layer.code.prepare_for_execution() + image_name = get_image_for_runtime(function_version.config.runtime) + if image_name not in PULLED_IMAGES: + CONTAINER_CLIENT.pull_image(image_name) + PULLED_IMAGES.add(image_name) + if config.LAMBDA_PREBUILD_IMAGES: + target_path = function_version.config.code.get_unzipped_code_location() + prepare_image(target_path, function_version) + LOG.debug( + "Version preparation of version %s took %0.2fms", + function_version.qualified_arn, + (time.perf_counter() - time_before) * 1000, + ) @classmethod def cleanup_version(cls, function_version: FunctionVersion) -> None: diff --git a/localstack/services/awslambda/invocation/lambda_models.py b/localstack/services/awslambda/invocation/lambda_models.py index feba6d3e11bb6..e8a6c85c90b00 100644 --- a/localstack/services/awslambda/invocation/lambda_models.py +++ b/localstack/services/awslambda/invocation/lambda_models.py @@ -490,6 +490,7 @@ class VersionFunctionConfiguration: last_modified: str # ISO string state: VersionState + image: Optional[str] = None image_config: Optional[ImageConfig] = None last_update: Optional[UpdateStatus] = None revision_id: str = dataclasses.field(init=False, default_factory=long_uid) diff --git a/localstack/services/awslambda/provider.py b/localstack/services/awslambda/provider.py index 38e1a812e4b03..e382a99b8aa81 100644 --- a/localstack/services/awslambda/provider.py +++ b/localstack/services/awslambda/provider.py @@ -146,6 +146,7 @@ FunctionResourcePolicy, FunctionUrlConfig, FunctionVersion, + ImageConfig, InvocationError, LambdaEphemeralStorage, Layer, @@ -498,8 +499,10 @@ def create_function( ) # save function code to s3 code = None + image = None + image_config = None + request_code = request.get("Code") if package_type == PackageType.Zip: - request_code = request["Code"] # TODO verify if correct combination of code is set if zip_file := request_code.get("ZipFile"): code = store_lambda_archive( @@ -521,6 +524,16 @@ def create_function( ) else: raise ServiceException("Gotta have s3 bucket or zip file") + elif package_type == PackageType.Image: + image = request_code.get("ImageUri") + if not image: + raise ServiceException("Gotta have an image when package type is image") + + image_config = ImageConfig( + command=request.get("ImageConfig", {}).get("Command"), + entrypoint=request.get("ImageConfig", {}).get("EntryPoint"), + working_directory=request.get("ImageConfig", {}).get("WorkingDirectory"), + ) version = FunctionVersion( id=arn, @@ -529,15 +542,16 @@ def create_function( description=request.get("Description", ""), role=request["Role"], timeout=request.get("Timeout", LAMBDA_DEFAULT_TIMEOUT), - runtime=request["Runtime"], + runtime=request.get("Runtime"), memory_size=request.get("MemorySize", LAMBDA_DEFAULT_MEMORY_SIZE), - handler=request["Handler"], - package_type=PackageType.Zip, # TODO + handler=request.get("Handler"), + package_type=package_type, reserved_concurrent_executions=0, environment=env_vars, architectures=request.get("Architectures") or ["x86_64"], # TODO tracing_config_mode=TracingMode.PassThrough, # TODO - image_config=None, # TODO + image=image, + image_config=image_config, code=code, layers=self.map_layers(layers), internal_revision=short_uid(), @@ -670,6 +684,7 @@ def update_function_code( ) function = state.functions[function_name] # TODO verify if correct combination of code is set + image = None if zip_file := request.get("ZipFile"): code = store_lambda_archive( archive_file=zip_file, @@ -688,13 +703,16 @@ def update_function_code( region_name=context.region, account_id=context.account_id, ) + elif image := request.get("ImageUri"): + code = None + image = image else: - raise ServiceException("Gotta have s3 bucket or zip file") + raise ServiceException("Gotta have s3 bucket or zip file or image") old_function_version = function.versions.get("$LATEST") + replace_kwargs = {"code": code} if code else {"image": image} config = dataclasses.replace( old_function_version.config, - code=code, internal_revision=short_uid(), last_modified=api_utils.generate_lambda_date(), last_update=UpdateStatus( @@ -702,6 +720,7 @@ def update_function_code( code="Creating", reason="The function is being created.", ), + **replace_kwargs, ) function_version = dataclasses.replace(old_function_version, config=config) function.versions["$LATEST"] = function_version @@ -756,7 +775,8 @@ def delete_function( for version in function.versions.values(): self.lambda_service.stop_version(qualified_arn=version.id.qualified_arn()) # we can safely destroy the code here - version.config.code.destroy() + if version.config.code: + version.config.code.destroy() def list_functions( self, @@ -819,12 +839,17 @@ def get_function( if tags: additional_fields["Tags"] = tags # TODO what if no version? - code = version.config.code + code_location = None + if code := version.config.code: + code_location = FunctionCodeLocation( + Location=code.generate_presigned_url(), RepositoryType="S3" + ) + elif image := version.config.image: + code_location = FunctionCodeLocation(ImageUri=image, ResolvedImageUri=image) + return GetFunctionResponse( Configuration=api_utils.map_config_out(version, return_qualified_arn=bool(qualifier)), - Code=FunctionCodeLocation( - Location=code.generate_presigned_url(), RepositoryType="S3" - ), # TODO + Code=code_location, # TODO **additional_fields # Concurrency={}, # TODO ) diff --git a/localstack/utils/container_utils/container_client.py b/localstack/utils/container_utils/container_client.py index 00d498e57e5e4..e7a84a415b715 100644 --- a/localstack/utils/container_utils/container_client.py +++ b/localstack/utils/container_utils/container_client.py @@ -773,6 +773,17 @@ def start_container( """ pass + @abstractmethod + def login(self, username: str, password: str, registry: Optional[str] = None) -> None: + """ + Login into an OCI registry + + :param username: Username for the registry + :param password: Password / token for the registry + :param registry: Registry url + """ + pass + class Util: MAX_ENV_ARGS_LENGTH = 20000 diff --git a/localstack/utils/container_utils/docker_cmd_client.py b/localstack/utils/container_utils/docker_cmd_client.py index ab28195a4fdbe..b3b1b3850cbb4 100644 --- a/localstack/utils/container_utils/docker_cmd_client.py +++ b/localstack/utils/container_utils/docker_cmd_client.py @@ -462,6 +462,19 @@ def get_container_ip(self, container_name_or_id: str) -> str: "Docker process returned with errorcode %s" % e.returncode, e.stdout, e.stderr ) from e + def login(self, username: str, password: str, registry: Optional[str] = None) -> None: + cmd = self._docker_cmd() + # TODO specify password via stdin + cmd += ["login", "-u", username, "-p", password] + if registry: + cmd.append(registry) + try: + run(cmd) + except subprocess.CalledProcessError as e: + raise ContainerException( + "Docker process returned with errorcode %s" % e.returncode, e.stdout, e.stderr + ) from e + def has_docker(self) -> bool: try: run(self._docker_cmd() + ["ps"]) diff --git a/localstack/utils/container_utils/docker_sdk_client.py b/localstack/utils/container_utils/docker_sdk_client.py index c3e67fe174ac5..3cd3a798cd02f 100644 --- a/localstack/utils/container_utils/docker_sdk_client.py +++ b/localstack/utils/container_utils/docker_sdk_client.py @@ -679,3 +679,10 @@ def exec_in_container( raise NoSuchContainer(container_name_or_id) except APIError as e: raise ContainerException() from e + + def login(self, username: str, password: str, registry: Optional[str] = None) -> None: + LOG.debug("Docker login for %s", username) + try: + self.client().login(username, password=password, registry=registry, reauth=True) + except APIError as e: + raise ContainerException() from e From bacf49dccd1577640c13ec6edb5268357e5a6a54 Mon Sep 17 00:00:00 2001 From: Daniel Fangl Date: Wed, 30 Nov 2022 21:37:27 +0100 Subject: [PATCH 2/5] add additional lambda image crud tests --- localstack/testing/pytest/fixtures.py | 6 + .../integration/awslambda/test_lambda_api.py | 178 +++++ .../awslambda/test_lambda_api.snapshot.json | 638 ++++++++++++++++++ 3 files changed, 822 insertions(+) diff --git a/localstack/testing/pytest/fixtures.py b/localstack/testing/pytest/fixtures.py index 674744cd9554f..6d4e14b3e7f3e 100644 --- a/localstack/testing/pytest/fixtures.py +++ b/localstack/testing/pytest/fixtures.py @@ -51,6 +51,7 @@ from mypy_boto3_dynamodb import DynamoDBClient, DynamoDBServiceResource from mypy_boto3_dynamodbstreams import DynamoDBStreamsClient from mypy_boto3_ec2 import EC2Client + from mypy_boto3_ecr import ECRClient from mypy_boto3_es import ElasticsearchServiceClient from mypy_boto3_events import EventBridgeClient from mypy_boto3_firehose import FirehoseClient @@ -401,6 +402,11 @@ def transcribe_client() -> "TranscribeClient": return _client("transcribe") +@pytest.fixture(scope="class") +def ecr_client() -> "ECRClient": + return _client("ecr") + + @pytest.fixture def dynamodb_wait_for_table_active(dynamodb_client): def wait_for_table_active(table_name: str, client=None): diff --git a/tests/integration/awslambda/test_lambda_api.py b/tests/integration/awslambda/test_lambda_api.py index 2204d383f7e31..352dd43df01cd 100644 --- a/tests/integration/awslambda/test_lambda_api.py +++ b/tests/integration/awslambda/test_lambda_api.py @@ -10,6 +10,7 @@ import base64 import io import json +import logging import os from hashlib import sha256 from io import BytesIO @@ -22,9 +23,11 @@ from localstack.aws.api.lambda_ import Architecture, Runtime from localstack.testing.aws.lambda_utils import _await_dynamodb_table_active, is_old_provider +from localstack.testing.aws.util import is_aws_cloud from localstack.testing.snapshots.transformer import SortingTransformer from localstack.utils import testutil from localstack.utils.aws import arns +from localstack.utils.docker_utils import DOCKER_CLIENT from localstack.utils.files import load_file from localstack.utils.strings import long_uid, short_uid, to_str from localstack.utils.sync import wait_until @@ -36,6 +39,8 @@ TEST_LAMBDA_PYTHON_VERSION, ) +LOG = logging.getLogger(__name__) + KB = 1024 @@ -523,6 +528,179 @@ def test_list_functions(self, lambda_client, create_lambda_function, lambda_su_r snapshot.match("list_default", list_default) +@pytest.mark.skipif(is_old_provider(), reason="focusing on new provider") +class TestLambdaImages: + @pytest.fixture(scope="class") + def login_docker_client(self, ecr_client): + if not is_aws_cloud(): + return + auth_data = ecr_client.get_authorization_token() + # if check is necessary since registry login data is not available at LS before min. 1 repository is created + if auth_data["authorizationData"]: + auth_data = auth_data["authorizationData"][0] + decoded_auth_token = str( + base64.decodebytes(bytes(auth_data["authorizationToken"], "utf-8")), "utf-8" + ) + username, password = decoded_auth_token.split(":") + DOCKER_CLIENT.login( + username=username, password=password, registry=auth_data["proxyEndpoint"] + ) + + @pytest.fixture(scope="class") + def test_image(self, ecr_client, login_docker_client): + repository_names = [] + image_names = [] + + def _create_test_image(base_image: str): + if is_aws_cloud(): + repository_name = f"test-repo-{short_uid()}" + repository_uri = ecr_client.create_repository(repositoryName=repository_name)[ + "repository" + ]["repositoryUri"] + image_name = f"{repository_uri}:latest" + repository_names.append(repository_name) + else: + image_name = f"test-image-{short_uid()}:latest" + image_names.append(image_name) + + DOCKER_CLIENT.pull_image(base_image) + DOCKER_CLIENT.tag_image(base_image, image_name) + if is_aws_cloud(): + DOCKER_CLIENT.push_image(image_name) + return image_name + + yield _create_test_image + + for image_name in image_names: + try: + DOCKER_CLIENT.remove_image(image=image_name, force=True) + except Exception as e: + LOG.debug("Error cleaning up image %s: %s", image_name, e) + + for repository_name in repository_names: + try: + image_ids = ecr_client.list_images(repositoryName=repository_name)["imageIds"] + ecr_client.batch_delete_image(repositoryName=repository_name, imageIds=image_ids) + ecr_client.delete_repository(repositoryName=repository_name) + except Exception as e: + LOG.debug("Error cleaning up repository %s: %s", repository_name, e) + + @pytest.mark.aws_validated + def test_lambda_image_crud( + self, lambda_client, create_lambda_function_aws, lambda_su_role, test_image, snapshot + ): + image = test_image("alpine") + repo_uri = image.rpartition(":")[0] + snapshot.add_transformer(snapshot.transform.regex(repo_uri, "")) + function_name = f"test-function-{short_uid()}" + create_image_response = create_lambda_function_aws( + FunctionName=function_name, + Role=lambda_su_role, + Code={"ImageUri": image}, + PackageType="Image", + Environment={"Variables": {"CUSTOM_ENV": "test"}}, + ) + snapshot.match("create-image-response", create_image_response) + lambda_client.get_waiter("function_active_v2").wait(FunctionName=function_name) + get_function_response = lambda_client.get_function(FunctionName=function_name) + snapshot.match("get-function-code-response", get_function_response) + get_function_config_response = lambda_client.get_function_configuration( + FunctionName=function_name + ) + snapshot.match("get-function-config-response", get_function_config_response) + + # try update to a zip file - should fail + with pytest.raises(ClientError) as e: + lambda_client.update_function_code( + FunctionName=function_name, + ZipFile=create_lambda_archive(load_file(TEST_LAMBDA_PYTHON_ECHO), get_content=True), + ) + snapshot.match("image-to-zipfile-error", e.value.response) + + image_2 = test_image("debian") + repo_uri_2 = image_2.rpartition(":")[0] + snapshot.add_transformer(snapshot.transform.regex(repo_uri_2, "")) + update_function_code_response = lambda_client.update_function_code( + FunctionName=function_name, ImageUri=image_2 + ) + snapshot.match("update-function-code-response", update_function_code_response) + lambda_client.get_waiter("function_updated_v2").wait(FunctionName=function_name) + + get_function_response = lambda_client.get_function(FunctionName=function_name) + snapshot.match("get-function-code-response-after-update", get_function_response) + get_function_config_response = lambda_client.get_function_configuration( + FunctionName=function_name + ) + snapshot.match("get-function-config-response-after-update", get_function_config_response) + + @pytest.mark.aws_validated + def test_lambda_image_and_image_config_crud( + self, lambda_client, create_lambda_function_aws, lambda_su_role, test_image, snapshot + ): + image = test_image("alpine") + repo_uri = image.rpartition(":")[0] + snapshot.add_transformer(snapshot.transform.regex(repo_uri, "")) + # Create another lambda with image config + function_name = f"test-function-{short_uid()}" + image_config = { + "EntryPoint": ["sh"], + "Command": ["-c", "echo test"], + "WorkingDirectory": "/app1", + } + create_image_response = create_lambda_function_aws( + FunctionName=function_name, + Role=lambda_su_role, + Code={"ImageUri": image}, + PackageType="Image", + ImageConfig=image_config, + Environment={"Variables": {"CUSTOM_ENV": "test"}}, + ) + snapshot.match("create-image-with-config-response", create_image_response) + lambda_client.get_waiter("function_active_v2").wait(FunctionName=function_name) + get_function_response = lambda_client.get_function(FunctionName=function_name) + snapshot.match("get-function-code-with-config-response", get_function_response) + get_function_config_response = lambda_client.get_function_configuration( + FunctionName=function_name + ) + snapshot.match("get-function-config-with-config-response", get_function_config_response) + + # update image config + new_image_config = { + "Command": ["-c", "echo test1"], + "WorkingDirectory": "/app1", + } + update_function_config_response = lambda_client.update_function_configuration( + FunctionName=function_name, ImageConfig=new_image_config + ) + snapshot.match("update-function-code-response", update_function_config_response) + lambda_client.get_waiter("function_updated_v2").wait(FunctionName=function_name) + + get_function_response = lambda_client.get_function(FunctionName=function_name) + snapshot.match("get-function-code-response-after-update", get_function_response) + get_function_config_response = lambda_client.get_function_configuration( + FunctionName=function_name + ) + snapshot.match("get-function-config-response-after-update", get_function_config_response) + + # update to empty image config + update_function_config_response = lambda_client.update_function_configuration( + FunctionName=function_name, ImageConfig={} + ) + snapshot.match( + "update-function-code-delete-imageconfig-response", update_function_config_response + ) + lambda_client.get_waiter("function_updated_v2").wait(FunctionName=function_name) + + get_function_response = lambda_client.get_function(FunctionName=function_name) + snapshot.match("get-function-code-response-after-delete-imageconfig", get_function_response) + get_function_config_response = lambda_client.get_function_configuration( + FunctionName=function_name + ) + snapshot.match( + "get-function-config-response-after-delete-imageconfig", get_function_config_response + ) + + @pytest.mark.skipif(is_old_provider(), reason="focusing on new provider") class TestLambdaVersions: @pytest.mark.aws_validated diff --git a/tests/integration/awslambda/test_lambda_api.snapshot.json b/tests/integration/awslambda/test_lambda_api.snapshot.json index c1f6a098bb9b3..af49c8d6e9f46 100644 --- a/tests/integration/awslambda/test_lambda_api.snapshot.json +++ b/tests/integration/awslambda/test_lambda_api.snapshot.json @@ -7954,5 +7954,643 @@ } } } + }, + "tests/integration/awslambda/test_lambda_api.py::TestLambdaImages::test_lambda_image_crud": { + "recorded-date": "30-11-2022, 20:38:27", + "recorded-content": { + "create-image-response": { + "Architectures": [ + "x86_64" + ], + "CodeSha256": "", + "CodeSize": 0, + "Description": "", + "Environment": { + "Variables": { + "CUSTOM_ENV": "test" + } + }, + "EphemeralStorage": { + "Size": 512 + }, + "FunctionArn": "arn:aws:lambda::111111111111:function:", + "FunctionName": "", + "LastModified": "date", + "MemorySize": 128, + "PackageType": "Image", + "RevisionId": "", + "Role": "arn:aws:iam::111111111111:role/", + "State": "Pending", + "StateReason": "The function is being created.", + "StateReasonCode": "Creating", + "Timeout": 3, + "TracingConfig": { + "Mode": "PassThrough" + }, + "Version": "$LATEST", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 201 + } + }, + "get-function-code-response": { + "Code": { + "ImageUri": ":latest", + "RepositoryType": "ECR", + "ResolvedImageUri": "@sha256:" + }, + "Configuration": { + "Architectures": [ + "x86_64" + ], + "CodeSha256": "", + "CodeSize": 0, + "Description": "", + "Environment": { + "Variables": { + "CUSTOM_ENV": "test" + } + }, + "EphemeralStorage": { + "Size": 512 + }, + "FunctionArn": "arn:aws:lambda::111111111111:function:", + "FunctionName": "", + "LastModified": "date", + "LastUpdateStatus": "Successful", + "MemorySize": 128, + "PackageType": "Image", + "RevisionId": "", + "Role": "arn:aws:iam::111111111111:role/", + "State": "Active", + "Timeout": 3, + "TracingConfig": { + "Mode": "PassThrough" + }, + "Version": "$LATEST" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get-function-config-response": { + "Architectures": [ + "x86_64" + ], + "CodeSha256": "", + "CodeSize": 0, + "Description": "", + "Environment": { + "Variables": { + "CUSTOM_ENV": "test" + } + }, + "EphemeralStorage": { + "Size": 512 + }, + "FunctionArn": "arn:aws:lambda::111111111111:function:", + "FunctionName": "", + "LastModified": "date", + "LastUpdateStatus": "Successful", + "MemorySize": 128, + "PackageType": "Image", + "RevisionId": "", + "Role": "arn:aws:iam::111111111111:role/", + "State": "Active", + "Timeout": 3, + "TracingConfig": { + "Mode": "PassThrough" + }, + "Version": "$LATEST", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "image-to-zipfile-error": { + "Error": { + "Code": "InvalidParameterValueException", + "Message": "Please provide ImageUri when updating a function with packageType Image." + }, + "Type": "User", + "message": "Please provide ImageUri when updating a function with packageType Image.", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 400 + } + }, + "update-function-code-response": { + "Architectures": [ + "x86_64" + ], + "CodeSha256": "", + "CodeSize": 0, + "Description": "", + "Environment": { + "Variables": { + "CUSTOM_ENV": "test" + } + }, + "EphemeralStorage": { + "Size": 512 + }, + "FunctionArn": "arn:aws:lambda::111111111111:function:", + "FunctionName": "", + "LastModified": "date", + "LastUpdateStatus": "InProgress", + "LastUpdateStatusReason": "The function is being created.", + "LastUpdateStatusReasonCode": "Creating", + "MemorySize": 128, + "PackageType": "Image", + "RevisionId": "", + "Role": "arn:aws:iam::111111111111:role/", + "State": "Active", + "Timeout": 3, + "TracingConfig": { + "Mode": "PassThrough" + }, + "Version": "$LATEST", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get-function-code-response-after-update": { + "Code": { + "ImageUri": ":latest", + "RepositoryType": "ECR", + "ResolvedImageUri": "@sha256:" + }, + "Configuration": { + "Architectures": [ + "x86_64" + ], + "CodeSha256": "", + "CodeSize": 0, + "Description": "", + "Environment": { + "Variables": { + "CUSTOM_ENV": "test" + } + }, + "EphemeralStorage": { + "Size": 512 + }, + "FunctionArn": "arn:aws:lambda::111111111111:function:", + "FunctionName": "", + "LastModified": "date", + "LastUpdateStatus": "Successful", + "MemorySize": 128, + "PackageType": "Image", + "RevisionId": "", + "Role": "arn:aws:iam::111111111111:role/", + "State": "Active", + "Timeout": 3, + "TracingConfig": { + "Mode": "PassThrough" + }, + "Version": "$LATEST" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get-function-config-response-after-update": { + "Architectures": [ + "x86_64" + ], + "CodeSha256": "", + "CodeSize": 0, + "Description": "", + "Environment": { + "Variables": { + "CUSTOM_ENV": "test" + } + }, + "EphemeralStorage": { + "Size": 512 + }, + "FunctionArn": "arn:aws:lambda::111111111111:function:", + "FunctionName": "", + "LastModified": "date", + "LastUpdateStatus": "Successful", + "MemorySize": 128, + "PackageType": "Image", + "RevisionId": "", + "Role": "arn:aws:iam::111111111111:role/", + "State": "Active", + "Timeout": 3, + "TracingConfig": { + "Mode": "PassThrough" + }, + "Version": "$LATEST", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + } + } + }, + "tests/integration/awslambda/test_lambda_api.py::TestLambdaImages::test_lambda_image_and_image_config_crud": { + "recorded-date": "30-11-2022, 21:35:01", + "recorded-content": { + "create-image-with-config-response": { + "Architectures": [ + "x86_64" + ], + "CodeSha256": "", + "CodeSize": 0, + "Description": "", + "Environment": { + "Variables": { + "CUSTOM_ENV": "test" + } + }, + "EphemeralStorage": { + "Size": 512 + }, + "FunctionArn": "arn:aws:lambda::111111111111:function:", + "FunctionName": "", + "ImageConfigResponse": { + "ImageConfig": { + "Command": [ + "-c", + "echo test" + ], + "EntryPoint": [ + "sh" + ], + "WorkingDirectory": "/app1" + } + }, + "LastModified": "date", + "MemorySize": 128, + "PackageType": "Image", + "RevisionId": "", + "Role": "arn:aws:iam::111111111111:role/", + "State": "Pending", + "StateReason": "The function is being created.", + "StateReasonCode": "Creating", + "Timeout": 3, + "TracingConfig": { + "Mode": "PassThrough" + }, + "Version": "$LATEST", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 201 + } + }, + "get-function-code-with-config-response": { + "Code": { + "ImageUri": ":latest", + "RepositoryType": "ECR", + "ResolvedImageUri": "@sha256:" + }, + "Configuration": { + "Architectures": [ + "x86_64" + ], + "CodeSha256": "", + "CodeSize": 0, + "Description": "", + "Environment": { + "Variables": { + "CUSTOM_ENV": "test" + } + }, + "EphemeralStorage": { + "Size": 512 + }, + "FunctionArn": "arn:aws:lambda::111111111111:function:", + "FunctionName": "", + "ImageConfigResponse": { + "ImageConfig": { + "Command": [ + "-c", + "echo test" + ], + "EntryPoint": [ + "sh" + ], + "WorkingDirectory": "/app1" + } + }, + "LastModified": "date", + "LastUpdateStatus": "Successful", + "MemorySize": 128, + "PackageType": "Image", + "RevisionId": "", + "Role": "arn:aws:iam::111111111111:role/", + "State": "Active", + "Timeout": 3, + "TracingConfig": { + "Mode": "PassThrough" + }, + "Version": "$LATEST" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get-function-config-with-config-response": { + "Architectures": [ + "x86_64" + ], + "CodeSha256": "", + "CodeSize": 0, + "Description": "", + "Environment": { + "Variables": { + "CUSTOM_ENV": "test" + } + }, + "EphemeralStorage": { + "Size": 512 + }, + "FunctionArn": "arn:aws:lambda::111111111111:function:", + "FunctionName": "", + "ImageConfigResponse": { + "ImageConfig": { + "Command": [ + "-c", + "echo test" + ], + "EntryPoint": [ + "sh" + ], + "WorkingDirectory": "/app1" + } + }, + "LastModified": "date", + "LastUpdateStatus": "Successful", + "MemorySize": 128, + "PackageType": "Image", + "RevisionId": "", + "Role": "arn:aws:iam::111111111111:role/", + "State": "Active", + "Timeout": 3, + "TracingConfig": { + "Mode": "PassThrough" + }, + "Version": "$LATEST", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "update-function-code-response": { + "Architectures": [ + "x86_64" + ], + "CodeSha256": "", + "CodeSize": 0, + "Description": "", + "Environment": { + "Variables": { + "CUSTOM_ENV": "test" + } + }, + "EphemeralStorage": { + "Size": 512 + }, + "FunctionArn": "arn:aws:lambda::111111111111:function:", + "FunctionName": "", + "ImageConfigResponse": { + "ImageConfig": { + "Command": [ + "-c", + "echo test1" + ], + "WorkingDirectory": "/app1" + } + }, + "LastModified": "date", + "LastUpdateStatus": "InProgress", + "LastUpdateStatusReason": "The function is being created.", + "LastUpdateStatusReasonCode": "Creating", + "MemorySize": 128, + "PackageType": "Image", + "RevisionId": "", + "Role": "arn:aws:iam::111111111111:role/", + "State": "Active", + "Timeout": 3, + "TracingConfig": { + "Mode": "PassThrough" + }, + "Version": "$LATEST", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get-function-code-response-after-update": { + "Code": { + "ImageUri": ":latest", + "RepositoryType": "ECR", + "ResolvedImageUri": "@sha256:" + }, + "Configuration": { + "Architectures": [ + "x86_64" + ], + "CodeSha256": "", + "CodeSize": 0, + "Description": "", + "Environment": { + "Variables": { + "CUSTOM_ENV": "test" + } + }, + "EphemeralStorage": { + "Size": 512 + }, + "FunctionArn": "arn:aws:lambda::111111111111:function:", + "FunctionName": "", + "ImageConfigResponse": { + "ImageConfig": { + "Command": [ + "-c", + "echo test1" + ], + "WorkingDirectory": "/app1" + } + }, + "LastModified": "date", + "LastUpdateStatus": "Successful", + "MemorySize": 128, + "PackageType": "Image", + "RevisionId": "", + "Role": "arn:aws:iam::111111111111:role/", + "State": "Active", + "Timeout": 3, + "TracingConfig": { + "Mode": "PassThrough" + }, + "Version": "$LATEST" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get-function-config-response-after-update": { + "Architectures": [ + "x86_64" + ], + "CodeSha256": "", + "CodeSize": 0, + "Description": "", + "Environment": { + "Variables": { + "CUSTOM_ENV": "test" + } + }, + "EphemeralStorage": { + "Size": 512 + }, + "FunctionArn": "arn:aws:lambda::111111111111:function:", + "FunctionName": "", + "ImageConfigResponse": { + "ImageConfig": { + "Command": [ + "-c", + "echo test1" + ], + "WorkingDirectory": "/app1" + } + }, + "LastModified": "date", + "LastUpdateStatus": "Successful", + "MemorySize": 128, + "PackageType": "Image", + "RevisionId": "", + "Role": "arn:aws:iam::111111111111:role/", + "State": "Active", + "Timeout": 3, + "TracingConfig": { + "Mode": "PassThrough" + }, + "Version": "$LATEST", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "update-function-code-delete-imageconfig-response": { + "Architectures": [ + "x86_64" + ], + "CodeSha256": "", + "CodeSize": 0, + "Description": "", + "Environment": { + "Variables": { + "CUSTOM_ENV": "test" + } + }, + "EphemeralStorage": { + "Size": 512 + }, + "FunctionArn": "arn:aws:lambda::111111111111:function:", + "FunctionName": "", + "LastModified": "date", + "LastUpdateStatus": "InProgress", + "LastUpdateStatusReason": "The function is being created.", + "LastUpdateStatusReasonCode": "Creating", + "MemorySize": 128, + "PackageType": "Image", + "RevisionId": "", + "Role": "arn:aws:iam::111111111111:role/", + "State": "Active", + "Timeout": 3, + "TracingConfig": { + "Mode": "PassThrough" + }, + "Version": "$LATEST", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get-function-code-response-after-delete-imageconfig": { + "Code": { + "ImageUri": ":latest", + "RepositoryType": "ECR", + "ResolvedImageUri": "@sha256:" + }, + "Configuration": { + "Architectures": [ + "x86_64" + ], + "CodeSha256": "", + "CodeSize": 0, + "Description": "", + "Environment": { + "Variables": { + "CUSTOM_ENV": "test" + } + }, + "EphemeralStorage": { + "Size": 512 + }, + "FunctionArn": "arn:aws:lambda::111111111111:function:", + "FunctionName": "", + "LastModified": "date", + "LastUpdateStatus": "Successful", + "MemorySize": 128, + "PackageType": "Image", + "RevisionId": "", + "Role": "arn:aws:iam::111111111111:role/", + "State": "Active", + "Timeout": 3, + "TracingConfig": { + "Mode": "PassThrough" + }, + "Version": "$LATEST" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get-function-config-response-after-delete-imageconfig": { + "Architectures": [ + "x86_64" + ], + "CodeSha256": "", + "CodeSize": 0, + "Description": "", + "Environment": { + "Variables": { + "CUSTOM_ENV": "test" + } + }, + "EphemeralStorage": { + "Size": 512 + }, + "FunctionArn": "arn:aws:lambda::111111111111:function:", + "FunctionName": "", + "LastModified": "date", + "LastUpdateStatus": "Successful", + "MemorySize": 128, + "PackageType": "Image", + "RevisionId": "", + "Role": "arn:aws:iam::111111111111:role/", + "State": "Active", + "Timeout": 3, + "TracingConfig": { + "Mode": "PassThrough" + }, + "Version": "$LATEST", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + } + } } } From dd1beeb0e20ec96c0e8e1497d16aa6a1c1204591 Mon Sep 17 00:00:00 2001 From: Daniel Fangl Date: Thu, 1 Dec 2022 18:16:20 +0100 Subject: [PATCH 3/5] add crud tests + proper crud implementation for lambda images --- localstack/services/awslambda/api_utils.py | 18 +- .../awslambda/invocation/lambda_models.py | 13 +- .../awslambda/invocation/lambda_service.py | 26 +++ localstack/services/awslambda/provider.py | 42 +++- .../integration/awslambda/test_lambda_api.py | 42 ++++ .../awslambda/test_lambda_api.snapshot.json | 185 ++++++++++++++++++ 6 files changed, 311 insertions(+), 15 deletions(-) diff --git a/localstack/services/awslambda/api_utils.py b/localstack/services/awslambda/api_utils.py index 9e12b8b800dd1..f05354334f51a 100644 --- a/localstack/services/awslambda/api_utils.py +++ b/localstack/services/awslambda/api_utils.py @@ -14,6 +14,8 @@ EphemeralStorage, FunctionConfiguration, FunctionUrlAuthType, + ImageConfig, + ImageConfigResponse, InvalidParameterValueException, LayerVersionContentOutput, PublishLayerVersionResponse, @@ -375,19 +377,21 @@ def map_config_out( for layer in version.config.layers ] if version.config.image_config: - image_config_response = {} + image_config = ImageConfig() if version.config.image_config.command: - image_config_response["Command"] = version.config.image_config.command + image_config["Command"] = version.config.image_config.command if version.config.image_config.entrypoint: - image_config_response["EntryPoint"] = version.config.image_config.entrypoint + image_config["EntryPoint"] = version.config.image_config.entrypoint if version.config.image_config.working_directory: - image_config_response[ - "WorkingDirectory" - ] = version.config.image_config.working_directory - optional_kwargs["ImageConfigResponse"] = image_config_response + image_config["WorkingDirectory"] = version.config.image_config.working_directory + if image_config: + optional_kwargs["ImageConfigResponse"] = ImageConfigResponse(ImageConfig=image_config) if version.config.code: optional_kwargs["CodeSize"] = version.config.code.code_size optional_kwargs["CodeSha256"] = version.config.code.code_sha256 + elif version.config.image: + optional_kwargs["CodeSize"] = 0 + optional_kwargs["CodeSha256"] = version.config.image.code_sha256 func_conf = FunctionConfiguration( RevisionId=version.config.revision_id, diff --git a/localstack/services/awslambda/invocation/lambda_models.py b/localstack/services/awslambda/invocation/lambda_models.py index e8a6c85c90b00..afcb4383e2859 100644 --- a/localstack/services/awslambda/invocation/lambda_models.py +++ b/localstack/services/awslambda/invocation/lambda_models.py @@ -203,6 +203,17 @@ def destroy(self) -> None: ) +@dataclasses.dataclass +class ImageCode: + image_uri: str + repository_type: str + code_sha256: str + + @property + def resolved_image_uri(self): + return f"{self.image_uri.rpartition(':')[0]}@sha256:{self.code_sha256}" + + @dataclasses.dataclass class DeadLetterConfig: target_arn: str @@ -490,7 +501,7 @@ class VersionFunctionConfiguration: last_modified: str # ISO string state: VersionState - image: Optional[str] = None + image: Optional[ImageCode] = None image_config: Optional[ImageConfig] = None last_update: Optional[UpdateStatus] = None revision_id: str = dataclasses.field(init=False, default_factory=long_uid) diff --git a/localstack/services/awslambda/invocation/lambda_service.py b/localstack/services/awslambda/invocation/lambda_service.py index 4daf02c8298c0..10f3e31de7c25 100644 --- a/localstack/services/awslambda/invocation/lambda_service.py +++ b/localstack/services/awslambda/invocation/lambda_service.py @@ -27,6 +27,7 @@ LAMBDA_LIMITS_CODE_SIZE_UNZIPPED_DEFAULT, Function, FunctionVersion, + ImageCode, Invocation, InvocationResult, S3Code, @@ -37,6 +38,8 @@ from localstack.services.awslambda.invocation.version_manager import LambdaVersionManager from localstack.utils.archives import get_unzipped_size, is_zip_file from localstack.utils.aws import aws_stack +from localstack.utils.container_utils.container_client import ContainerException +from localstack.utils.docker_utils import DOCKER_CLIENT as CONTAINER_CLIENT from localstack.utils.strings import to_str if TYPE_CHECKING: @@ -402,3 +405,26 @@ def store_s3_bucket_archive( return store_lambda_archive( archive_file, function_name=function_name, region_name=region_name, account_id=account_id ) + + +def store_image_code(image_uri: str) -> ImageCode: + """ + Creates an image code by inspecting the provided image + + :param image_uri: Image URI of the image to inspect + :return: Image code object + """ + code_sha256 = "" + try: + CONTAINER_CLIENT.pull_image(docker_image=image_uri) + except ContainerException: + LOG.debug("Cannot pull image %s. Maybe only available locally?", image_uri) + try: + code_sha256 = CONTAINER_CLIENT.inspect_image(image_name=image_uri)["RepoDigests"][ + 0 + ].partition(":")[2] + except Exception as e: + LOG.debug( + "Cannot inspect image %s. Is this image and/or docker available: %s", image_uri, e + ) + return ImageCode(image_uri=image_uri, code_sha256=code_sha256, repository_type="ECR") diff --git a/localstack/services/awslambda/provider.py b/localstack/services/awslambda/provider.py index e382a99b8aa81..806ab1977b3ab 100644 --- a/localstack/services/awslambda/provider.py +++ b/localstack/services/awslambda/provider.py @@ -168,6 +168,7 @@ LambdaService, destroy_code_if_not_used, lambda_stores, + store_image_code, store_lambda_archive, store_s3_bucket_archive, ) @@ -528,12 +529,14 @@ def create_function( image = request_code.get("ImageUri") if not image: raise ServiceException("Gotta have an image when package type is image") + image = store_image_code(image_uri=image) - image_config = ImageConfig( - command=request.get("ImageConfig", {}).get("Command"), - entrypoint=request.get("ImageConfig", {}).get("EntryPoint"), - working_directory=request.get("ImageConfig", {}).get("WorkingDirectory"), - ) + if image_config_req := request.get("ImageConfig"): + image_config = ImageConfig( + command=image_config_req.get("Command"), + entrypoint=image_config_req.get("EntryPoint"), + working_directory=image_config_req.get("WorkingDirectory"), + ) version = FunctionVersion( id=arn, @@ -647,6 +650,14 @@ def update_function_configuration( self._validate_layers(new_layers) replace_kwargs["layers"] = self.map_layers(new_layers) + if "ImageConfig" in request: + new_image_config = request["ImageConfig"] + replace_kwargs["image_config"] = ImageConfig( + command=new_image_config.get("Command"), + entrypoint=new_image_config.get("EntryPoint"), + working_directory=new_image_config.get("WorkingDirectory"), + ) + new_latest_version = dataclasses.replace( latest_version, config=dataclasses.replace( @@ -685,6 +696,19 @@ def update_function_code( function = state.functions[function_name] # TODO verify if correct combination of code is set image = None + if ( + request.get("ZipFile") or request.get("S3Bucket") + ) and function.latest().config.package_type == PackageType.Image: + raise InvalidParameterValueException( + "Please provide ImageUri when updating a function with packageType Image.", + Type="User", + ) + elif request.get("ImageUri") and function.latest().config.package_type == PackageType.Zip: + raise InvalidParameterValueException( + "Please don't provide ImageUri when updating a function with packageType Zip.", + Type="User", + ) + if zip_file := request.get("ZipFile"): code = store_lambda_archive( archive_file=zip_file, @@ -705,7 +729,7 @@ def update_function_code( ) elif image := request.get("ImageUri"): code = None - image = image + image = store_image_code(image_uri=image) else: raise ServiceException("Gotta have s3 bucket or zip file or image") @@ -845,7 +869,11 @@ def get_function( Location=code.generate_presigned_url(), RepositoryType="S3" ) elif image := version.config.image: - code_location = FunctionCodeLocation(ImageUri=image, ResolvedImageUri=image) + code_location = FunctionCodeLocation( + ImageUri=image.image_uri, + RepositoryType=image.repository_type, + ResolvedImageUri=image.resolved_image_uri, + ) return GetFunctionResponse( Configuration=api_utils.map_config_out(version, return_qualified_arn=bool(qualifier)), diff --git a/tests/integration/awslambda/test_lambda_api.py b/tests/integration/awslambda/test_lambda_api.py index 352dd43df01cd..6dbfa148e24bb 100644 --- a/tests/integration/awslambda/test_lambda_api.py +++ b/tests/integration/awslambda/test_lambda_api.py @@ -589,6 +589,7 @@ def _create_test_image(base_image: str): def test_lambda_image_crud( self, lambda_client, create_lambda_function_aws, lambda_su_role, test_image, snapshot ): + """Test lambda crud with package type image""" image = test_image("alpine") repo_uri = image.rpartition(":")[0] snapshot.add_transformer(snapshot.transform.regex(repo_uri, "")) @@ -633,10 +634,51 @@ def test_lambda_image_crud( ) snapshot.match("get-function-config-response-after-update", get_function_config_response) + @pytest.mark.aws_validated + def test_lambda_zip_file_to_image( + self, lambda_client, create_lambda_function_aws, lambda_su_role, test_image, snapshot + ): + """Test that verifies conversion from zip file lambda to image lambda is not possible""" + image = test_image("alpine") + repo_uri = image.rpartition(":")[0] + snapshot.add_transformer(snapshot.transform.regex(repo_uri, "")) + function_name = f"test-function-{short_uid()}" + create_image_response = create_lambda_function_aws( + FunctionName=function_name, + Role=lambda_su_role, + Runtime=Runtime.python3_9, + Handler="handler.handler", + Code={ + "ZipFile": create_lambda_archive( + load_file(TEST_LAMBDA_PYTHON_ECHO), get_content=True + ) + }, + ) + snapshot.match("create-image-response", create_image_response) + lambda_client.get_waiter("function_active_v2").wait(FunctionName=function_name) + get_function_response = lambda_client.get_function(FunctionName=function_name) + snapshot.match("get-function-code-response", get_function_response) + get_function_config_response = lambda_client.get_function_configuration( + FunctionName=function_name + ) + snapshot.match("get-function-config-response", get_function_config_response) + + with pytest.raises(ClientError) as e: + lambda_client.update_function_code(FunctionName=function_name, ImageUri=image) + snapshot.match("zipfile-to-image-error", e.value.response) + + get_function_response = lambda_client.get_function(FunctionName=function_name) + snapshot.match("get-function-code-response-after-update", get_function_response) + get_function_config_response = lambda_client.get_function_configuration( + FunctionName=function_name + ) + snapshot.match("get-function-config-response-after-update", get_function_config_response) + @pytest.mark.aws_validated def test_lambda_image_and_image_config_crud( self, lambda_client, create_lambda_function_aws, lambda_su_role, test_image, snapshot ): + """Test lambda crud with packagetype image and image configs""" image = test_image("alpine") repo_uri = image.rpartition(":")[0] snapshot.add_transformer(snapshot.transform.regex(repo_uri, "")) diff --git a/tests/integration/awslambda/test_lambda_api.snapshot.json b/tests/integration/awslambda/test_lambda_api.snapshot.json index af49c8d6e9f46..34d409b546710 100644 --- a/tests/integration/awslambda/test_lambda_api.snapshot.json +++ b/tests/integration/awslambda/test_lambda_api.snapshot.json @@ -8592,5 +8592,190 @@ } } } + }, + "tests/integration/awslambda/test_lambda_api.py::TestLambdaImages::test_lambda_zip_file_to_image": { + "recorded-date": "01-12-2022, 13:34:38", + "recorded-content": { + "create-image-response": { + "Architectures": [ + "x86_64" + ], + "CodeSha256": "", + "CodeSize": "", + "Description": "", + "EphemeralStorage": { + "Size": 512 + }, + "FunctionArn": "arn:aws:lambda::111111111111:function:", + "FunctionName": "", + "Handler": "handler.handler", + "LastModified": "date", + "MemorySize": 128, + "PackageType": "Zip", + "RevisionId": "", + "Role": "arn:aws:iam::111111111111:role/", + "Runtime": "python3.9", + "State": "Pending", + "StateReason": "The function is being created.", + "StateReasonCode": "Creating", + "Timeout": 3, + "TracingConfig": { + "Mode": "PassThrough" + }, + "Version": "$LATEST", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 201 + } + }, + "get-function-code-response": { + "Code": { + "Location": "", + "RepositoryType": "S3" + }, + "Configuration": { + "Architectures": [ + "x86_64" + ], + "CodeSha256": "", + "CodeSize": "", + "Description": "", + "EphemeralStorage": { + "Size": 512 + }, + "FunctionArn": "arn:aws:lambda::111111111111:function:", + "FunctionName": "", + "Handler": "handler.handler", + "LastModified": "date", + "LastUpdateStatus": "Successful", + "MemorySize": 128, + "PackageType": "Zip", + "RevisionId": "", + "Role": "arn:aws:iam::111111111111:role/", + "Runtime": "python3.9", + "State": "Active", + "Timeout": 3, + "TracingConfig": { + "Mode": "PassThrough" + }, + "Version": "$LATEST" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get-function-config-response": { + "Architectures": [ + "x86_64" + ], + "CodeSha256": "", + "CodeSize": "", + "Description": "", + "EphemeralStorage": { + "Size": 512 + }, + "FunctionArn": "arn:aws:lambda::111111111111:function:", + "FunctionName": "", + "Handler": "handler.handler", + "LastModified": "date", + "LastUpdateStatus": "Successful", + "MemorySize": 128, + "PackageType": "Zip", + "RevisionId": "", + "Role": "arn:aws:iam::111111111111:role/", + "Runtime": "python3.9", + "State": "Active", + "Timeout": 3, + "TracingConfig": { + "Mode": "PassThrough" + }, + "Version": "$LATEST", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "zipfile-to-image-error": { + "Error": { + "Code": "InvalidParameterValueException", + "Message": "Please don't provide ImageUri when updating a function with packageType Zip." + }, + "Type": "User", + "message": "Please don't provide ImageUri when updating a function with packageType Zip.", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 400 + } + }, + "get-function-code-response-after-update": { + "Code": { + "Location": "", + "RepositoryType": "S3" + }, + "Configuration": { + "Architectures": [ + "x86_64" + ], + "CodeSha256": "", + "CodeSize": "", + "Description": "", + "EphemeralStorage": { + "Size": 512 + }, + "FunctionArn": "arn:aws:lambda::111111111111:function:", + "FunctionName": "", + "Handler": "handler.handler", + "LastModified": "date", + "LastUpdateStatus": "Successful", + "MemorySize": 128, + "PackageType": "Zip", + "RevisionId": "", + "Role": "arn:aws:iam::111111111111:role/", + "Runtime": "python3.9", + "State": "Active", + "Timeout": 3, + "TracingConfig": { + "Mode": "PassThrough" + }, + "Version": "$LATEST" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get-function-config-response-after-update": { + "Architectures": [ + "x86_64" + ], + "CodeSha256": "", + "CodeSize": "", + "Description": "", + "EphemeralStorage": { + "Size": 512 + }, + "FunctionArn": "arn:aws:lambda::111111111111:function:", + "FunctionName": "", + "Handler": "handler.handler", + "LastModified": "date", + "LastUpdateStatus": "Successful", + "MemorySize": 128, + "PackageType": "Zip", + "RevisionId": "", + "Role": "arn:aws:iam::111111111111:role/", + "Runtime": "python3.9", + "State": "Active", + "Timeout": 3, + "TracingConfig": { + "Mode": "PassThrough" + }, + "Version": "$LATEST", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + } + } } } From e0fffae1fd929b8cd6e81b8b7b935254f7d84080 Mon Sep 17 00:00:00 2001 From: Daniel Fangl Date: Thu, 1 Dec 2022 22:01:07 +0100 Subject: [PATCH 4/5] fix minor issues for images with port and presence of imageconfig in model --- .../services/awslambda/invocation/lambda_service.py | 2 +- localstack/services/awslambda/provider.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/localstack/services/awslambda/invocation/lambda_service.py b/localstack/services/awslambda/invocation/lambda_service.py index 10f3e31de7c25..303532ea5ea8b 100644 --- a/localstack/services/awslambda/invocation/lambda_service.py +++ b/localstack/services/awslambda/invocation/lambda_service.py @@ -422,7 +422,7 @@ def store_image_code(image_uri: str) -> ImageCode: try: code_sha256 = CONTAINER_CLIENT.inspect_image(image_name=image_uri)["RepoDigests"][ 0 - ].partition(":")[2] + ].rpartition(":")[2] except Exception as e: LOG.debug( "Cannot inspect image %s. Is this image and/or docker available: %s", image_uri, e diff --git a/localstack/services/awslambda/provider.py b/localstack/services/awslambda/provider.py index 806ab1977b3ab..de8ef2d2f80de 100644 --- a/localstack/services/awslambda/provider.py +++ b/localstack/services/awslambda/provider.py @@ -531,12 +531,12 @@ def create_function( raise ServiceException("Gotta have an image when package type is image") image = store_image_code(image_uri=image) - if image_config_req := request.get("ImageConfig"): - image_config = ImageConfig( - command=image_config_req.get("Command"), - entrypoint=image_config_req.get("EntryPoint"), - working_directory=image_config_req.get("WorkingDirectory"), - ) + image_config_req = request.get("ImageConfig", {}) + image_config = ImageConfig( + command=image_config_req.get("Command"), + entrypoint=image_config_req.get("EntryPoint"), + working_directory=image_config_req.get("WorkingDirectory"), + ) version = FunctionVersion( id=arn, From c6afe20aee6c08a2b2505c5583071258e8bca580 Mon Sep 17 00:00:00 2001 From: Daniel Fangl Date: Fri, 2 Dec 2022 10:28:14 +0100 Subject: [PATCH 5/5] make dataclasses for image config + image code frozen, move hooks, add cleanup --- localstack/services/awslambda/hooks.py | 7 +++++++ .../awslambda/invocation/docker_runtime_executor.py | 13 +++---------- .../services/awslambda/invocation/lambda_models.py | 4 ++-- .../services/awslambda/invocation/lambda_service.py | 2 +- localstack/services/awslambda/provider.py | 6 +++--- tests/integration/awslambda/test_lambda_api.py | 11 +++++++++-- 6 files changed, 25 insertions(+), 18 deletions(-) create mode 100644 localstack/services/awslambda/hooks.py diff --git a/localstack/services/awslambda/hooks.py b/localstack/services/awslambda/hooks.py new file mode 100644 index 0000000000000..02870e99e90a8 --- /dev/null +++ b/localstack/services/awslambda/hooks.py @@ -0,0 +1,7 @@ +from localstack.runtime.hooks import hook_spec + +HOOKS_LAMBDA_START_DOCKER_EXECUTOR = "localstack.hooks.lambda_start_docker_executor" +HOOKS_LAMBDA_PREPARE_DOCKER_EXECUTOR = "localstack.hooks.lambda_prepare_docker_executors" + +start_docker_executor = hook_spec(HOOKS_LAMBDA_START_DOCKER_EXECUTOR) +prepare_docker_executor = hook_spec(HOOKS_LAMBDA_PREPARE_DOCKER_EXECUTOR) diff --git a/localstack/services/awslambda/invocation/docker_runtime_executor.py b/localstack/services/awslambda/invocation/docker_runtime_executor.py index 8db7bbf086f84..2a2c1a0ec25db 100644 --- a/localstack/services/awslambda/invocation/docker_runtime_executor.py +++ b/localstack/services/awslambda/invocation/docker_runtime_executor.py @@ -8,7 +8,7 @@ from localstack import config from localstack.aws.api.lambda_ import PackageType -from localstack.runtime.hooks import hook_spec +from localstack.services.awslambda import hooks as lambda_hooks from localstack.services.awslambda.invocation.executor_endpoint import ( ExecutorEndpoint, ServiceEndpoint, @@ -27,13 +27,6 @@ from localstack.utils.docker_utils import DOCKER_CLIENT as CONTAINER_CLIENT from localstack.utils.strings import truncate -# Hook definitions -HOOKS_LAMBDA_START_DOCKER_EXECUTOR = "localstack.hooks.lambda_start_docker_executor" -HOOKS_LAMBDA_PREPARE_DOCKER_EXECUTOR = "localstack.hooks.lambda_prepare_docker_executors" - -start_docker_executor = hook_spec(HOOKS_LAMBDA_START_DOCKER_EXECUTOR) -prepare_docker_executor = hook_spec(HOOKS_LAMBDA_PREPARE_DOCKER_EXECUTOR) - LOG = logging.getLogger(__name__) RUNTIME_REGEX = r"(?P[a-z]+)(?P\d+(\.\d+)?(\.al2)?)(?:.*)" @@ -157,7 +150,7 @@ def start(self, env_vars: dict[str, str]) -> None: network=network, entrypoint=RAPID_ENTRYPOINT, ) - start_docker_executor.run(container_config, self.function_version) + lambda_hooks.start_docker_executor.run(container_config, self.function_version) if not container_config.image_name: container_config.image_name = self.get_image() @@ -220,7 +213,7 @@ def invoke(self, payload: Dict[str, str]): @classmethod def prepare_version(cls, function_version: FunctionVersion) -> None: time_before = time.perf_counter() - prepare_docker_executor.run(function_version) + lambda_hooks.prepare_docker_executor.run(function_version) if function_version.config.code: function_version.config.code.prepare_for_execution() for layer in function_version.config.layers: diff --git a/localstack/services/awslambda/invocation/lambda_models.py b/localstack/services/awslambda/invocation/lambda_models.py index afcb4383e2859..f09dc8869f65a 100644 --- a/localstack/services/awslambda/invocation/lambda_models.py +++ b/localstack/services/awslambda/invocation/lambda_models.py @@ -203,7 +203,7 @@ def destroy(self) -> None: ) -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class ImageCode: image_uri: str repository_type: str @@ -225,7 +225,7 @@ class FileSystemConfig: local_mount_path: str -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class ImageConfig: working_directory: str command: list[str] = dataclasses.field(default_factory=list) diff --git a/localstack/services/awslambda/invocation/lambda_service.py b/localstack/services/awslambda/invocation/lambda_service.py index 303532ea5ea8b..d8009f8062143 100644 --- a/localstack/services/awslambda/invocation/lambda_service.py +++ b/localstack/services/awslambda/invocation/lambda_service.py @@ -407,7 +407,7 @@ def store_s3_bucket_archive( ) -def store_image_code(image_uri: str) -> ImageCode: +def create_image_code(image_uri: str) -> ImageCode: """ Creates an image code by inspecting the provided image diff --git a/localstack/services/awslambda/provider.py b/localstack/services/awslambda/provider.py index de8ef2d2f80de..0b09c4fc8e49a 100644 --- a/localstack/services/awslambda/provider.py +++ b/localstack/services/awslambda/provider.py @@ -166,9 +166,9 @@ ) from localstack.services.awslambda.invocation.lambda_service import ( LambdaService, + create_image_code, destroy_code_if_not_used, lambda_stores, - store_image_code, store_lambda_archive, store_s3_bucket_archive, ) @@ -529,7 +529,7 @@ def create_function( image = request_code.get("ImageUri") if not image: raise ServiceException("Gotta have an image when package type is image") - image = store_image_code(image_uri=image) + image = create_image_code(image_uri=image) image_config_req = request.get("ImageConfig", {}) image_config = ImageConfig( @@ -729,7 +729,7 @@ def update_function_code( ) elif image := request.get("ImageUri"): code = None - image = store_image_code(image_uri=image) + image = create_image_code(image_uri=image) else: raise ServiceException("Gotta have s3 bucket or zip file or image") diff --git a/tests/integration/awslambda/test_lambda_api.py b/tests/integration/awslambda/test_lambda_api.py index 6dbfa148e24bb..c9324fa924112 100644 --- a/tests/integration/awslambda/test_lambda_api.py +++ b/tests/integration/awslambda/test_lambda_api.py @@ -29,6 +29,7 @@ from localstack.utils.aws import arns from localstack.utils.docker_utils import DOCKER_CLIENT from localstack.utils.files import load_file +from localstack.utils.functions import call_safe from localstack.utils.strings import long_uid, short_uid, to_str from localstack.utils.sync import wait_until from localstack.utils.testutil import create_lambda_archive @@ -579,8 +580,14 @@ def _create_test_image(base_image: str): for repository_name in repository_names: try: - image_ids = ecr_client.list_images(repositoryName=repository_name)["imageIds"] - ecr_client.batch_delete_image(repositoryName=repository_name, imageIds=image_ids) + image_ids = ecr_client.list_images(repositoryName=repository_name).get( + "imageIds", [] + ) + if image_ids: + call_safe( + ecr_client.batch_delete_image, + kwargs={"repositoryName": repository_name, "imageIds": image_ids}, + ) ecr_client.delete_repository(repositoryName=repository_name) except Exception as e: LOG.debug("Error cleaning up repository %s: %s", repository_name, e)