Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Implement Lambda ASF container image CRUD and docker runtime executor hooks #7266

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Dec 2, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions localstack/services/awslambda/api_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
EphemeralStorage,
FunctionConfiguration,
FunctionUrlAuthType,
ImageConfig,
ImageConfigResponse,
InvalidParameterValueException,
LayerVersionContentOutput,
PublishLayerVersionResponse,
Expand Down Expand Up @@ -374,6 +376,22 @@ def map_config_out(
{"Arn": layer.layer_version_arn, "CodeSize": layer.code.code_size}
for layer in version.config.layers
]
if version.config.image_config:
image_config = ImageConfig()
if version.config.image_config.command:
image_config["Command"] = version.config.image_config.command
if version.config.image_config.entrypoint:
image_config["EntryPoint"] = version.config.image_config.entrypoint
if version.config.image_config.working_directory:
image_config["WorkingDirectory"] = version.config.image_config.working_directory
if image_config:
optional_kwargs["ImageConfigResponse"] = ImageConfigResponse(ImageConfig=image_config)
if version.config.code:
optional_kwargs["CodeSize"] = version.config.code.code_size
optional_kwargs["CodeSha256"] = version.config.code.code_sha256
elif version.config.image:
optional_kwargs["CodeSize"] = 0
optional_kwargs["CodeSha256"] = version.config.image.code_sha256

func_conf = FunctionConfiguration(
RevisionId=version.config.revision_id,
Expand All @@ -388,8 +406,6 @@ def map_config_out(
Timeout=version.config.timeout,
Runtime=version.config.runtime,
Handler=version.config.handler,
CodeSize=version.config.code.code_size,
CodeSha256=version.config.code.code_sha256,
MemorySize=version.config.memory_size,
PackageType=version.config.package_type,
TracingConfig=TracingConfig(Mode=version.config.tracing_config_mode),
Expand Down
7 changes: 7 additions & 0 deletions localstack/services/awslambda/hooks.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
from localstack.runtime.hooks import hook_spec

HOOKS_LAMBDA_START_DOCKER_EXECUTOR = "localstack.hooks.lambda_start_docker_executor"
HOOKS_LAMBDA_PREPARE_DOCKER_EXECUTOR = "localstack.hooks.lambda_prepare_docker_executors"

start_docker_executor = hook_spec(HOOKS_LAMBDA_START_DOCKER_EXECUTOR)
prepare_docker_executor = hook_spec(HOOKS_LAMBDA_PREPARE_DOCKER_EXECUTOR)
66 changes: 45 additions & 21 deletions localstack/services/awslambda/invocation/docker_runtime_executor.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import dataclasses
import json
import logging
import shutil
Expand All @@ -6,6 +7,8 @@
from typing import Dict, Literal, Optional

from localstack import config
from localstack.aws.api.lambda_ import PackageType
from localstack.services.awslambda import hooks as lambda_hooks
from localstack.services.awslambda.invocation.executor_endpoint import (
ExecutorEndpoint,
ServiceEndpoint,
Expand Down Expand Up @@ -96,6 +99,11 @@ def prepare_image(target_path: Path, function_version: FunctionVersion) -> None:
)


@dataclasses.dataclass
class LambdaContainerConfiguration(ContainerConfiguration):
copy_folders: list[tuple[str, str]] = dataclasses.field(default_factory=list)


class DockerRuntimeExecutor(RuntimeExecutor):
ip: Optional[str]
executor_endpoint: Optional[ExecutorEndpoint]
Expand Down Expand Up @@ -135,23 +143,35 @@ def _build_executor_endpoint(self, service_endpoint: ServiceEndpoint) -> Executo
def start(self, env_vars: dict[str, str]) -> None:
self.executor_endpoint.start()
network = self._get_network_for_executor()
container_config = ContainerConfiguration(
image_name=self.get_image(),
container_config = LambdaContainerConfiguration(
image_name=None,
name=self.id,
env_vars=env_vars,
network=network,
entrypoint=RAPID_ENTRYPOINT,
)
lambda_hooks.start_docker_executor.run(container_config, self.function_version)

if not container_config.image_name:
container_config.image_name = self.get_image()

CONTAINER_CLIENT.create_container_from_config(container_config)
if not config.LAMBDA_PREBUILD_IMAGES:
if (
not config.LAMBDA_PREBUILD_IMAGES
or self.function_version.config.package_type != PackageType.Zip
):
CONTAINER_CLIENT.copy_into_container(
self.id, str(get_runtime_client_path()), RAPID_ENTRYPOINT
)
CONTAINER_CLIENT.copy_into_container(
self.id,
f"{str(self.function_version.config.code.get_unzipped_code_location())}/.",
"/var/task",
)
if self.function_version.config.package_type == PackageType.Zip:
if not config.LAMBDA_PREBUILD_IMAGES:
CONTAINER_CLIENT.copy_into_container(
self.id,
f"{str(self.function_version.config.code.get_unzipped_code_location())}/.",
"/var/task",
)
for source, target in container_config.copy_folders:
CONTAINER_CLIENT.copy_into_container(self.id, source, target)

CONTAINER_CLIENT.start_container(self.id)
self.ip = CONTAINER_CLIENT.get_container_ipv4_for_network(
Expand Down Expand Up @@ -193,19 +213,23 @@ def invoke(self, payload: Dict[str, str]):
@classmethod
def prepare_version(cls, function_version: FunctionVersion) -> None:
time_before = time.perf_counter()
function_version.config.code.prepare_for_execution()
target_path = function_version.config.code.get_unzipped_code_location()
image_name = get_image_for_runtime(function_version.config.runtime)
if image_name not in PULLED_IMAGES:
CONTAINER_CLIENT.pull_image(image_name)
PULLED_IMAGES.add(image_name)
if config.LAMBDA_PREBUILD_IMAGES:
prepare_image(target_path, function_version)
LOG.debug(
"Version preparation of version %s took %0.2fms",
function_version.qualified_arn,
(time.perf_counter() - time_before) * 1000,
)
lambda_hooks.prepare_docker_executor.run(function_version)
if function_version.config.code:
function_version.config.code.prepare_for_execution()
for layer in function_version.config.layers:
layer.code.prepare_for_execution()
image_name = get_image_for_runtime(function_version.config.runtime)
if image_name not in PULLED_IMAGES:
CONTAINER_CLIENT.pull_image(image_name)
PULLED_IMAGES.add(image_name)
if config.LAMBDA_PREBUILD_IMAGES:
target_path = function_version.config.code.get_unzipped_code_location()
prepare_image(target_path, function_version)
LOG.debug(
"Version preparation of version %s took %0.2fms",
function_version.qualified_arn,
(time.perf_counter() - time_before) * 1000,
)

@classmethod
def cleanup_version(cls, function_version: FunctionVersion) -> None:
Expand Down
14 changes: 13 additions & 1 deletion localstack/services/awslambda/invocation/lambda_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,17 @@ def destroy(self) -> None:
)


@dataclasses.dataclass(frozen=True)
class ImageCode:
image_uri: str
repository_type: str
code_sha256: str

@property
def resolved_image_uri(self):
return f"{self.image_uri.rpartition(':')[0]}@sha256:{self.code_sha256}"


@dataclasses.dataclass
class DeadLetterConfig:
target_arn: str
Expand All @@ -214,7 +225,7 @@ class FileSystemConfig:
local_mount_path: str


@dataclasses.dataclass
@dataclasses.dataclass(frozen=True)
class ImageConfig:
working_directory: str
command: list[str] = dataclasses.field(default_factory=list)
Expand Down Expand Up @@ -490,6 +501,7 @@ class VersionFunctionConfiguration:
last_modified: str # ISO string
state: VersionState

image: Optional[ImageCode] = None
image_config: Optional[ImageConfig] = None
last_update: Optional[UpdateStatus] = None
revision_id: str = dataclasses.field(init=False, default_factory=long_uid)
Expand Down
26 changes: 26 additions & 0 deletions localstack/services/awslambda/invocation/lambda_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
LAMBDA_LIMITS_CODE_SIZE_UNZIPPED_DEFAULT,
Function,
FunctionVersion,
ImageCode,
Invocation,
InvocationResult,
S3Code,
Expand All @@ -37,6 +38,8 @@
from localstack.services.awslambda.invocation.version_manager import LambdaVersionManager
from localstack.utils.archives import get_unzipped_size, is_zip_file
from localstack.utils.aws import aws_stack
from localstack.utils.container_utils.container_client import ContainerException
from localstack.utils.docker_utils import DOCKER_CLIENT as CONTAINER_CLIENT
from localstack.utils.strings import to_str

if TYPE_CHECKING:
Expand Down Expand Up @@ -402,3 +405,26 @@ def store_s3_bucket_archive(
return store_lambda_archive(
archive_file, function_name=function_name, region_name=region_name, account_id=account_id
)


def create_image_code(image_uri: str) -> ImageCode:
"""
Creates an image code by inspecting the provided image

:param image_uri: Image URI of the image to inspect
:return: Image code object
"""
code_sha256 = "<cannot-find-image>"
try:
CONTAINER_CLIENT.pull_image(docker_image=image_uri)
except ContainerException:
LOG.debug("Cannot pull image %s. Maybe only available locally?", image_uri)
try:
code_sha256 = CONTAINER_CLIENT.inspect_image(image_name=image_uri)["RepoDigests"][
0
].rpartition(":")[2]
except Exception as e:
LOG.debug(
"Cannot inspect image %s. Is this image and/or docker available: %s", image_uri, e
)
return ImageCode(image_uri=image_uri, code_sha256=code_sha256, repository_type="ECR")
Loading