From 304b73d1ee8fe90b51fed46209fb7b1610a10678 Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Wed, 21 May 2025 16:18:26 +0200 Subject: [PATCH 1/6] Resolve non-subdomain host prefixes to LocalStack --- localstack-core/localstack/dns/server.py | 24 +++++++++++++++++++++++- tests/unit/test_dns_server.py | 24 +++++++++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/localstack-core/localstack/dns/server.py b/localstack-core/localstack/dns/server.py index 6cf61ec0b0937..0e708cfa71a38 100644 --- a/localstack-core/localstack/dns/server.py +++ b/localstack-core/localstack/dns/server.py @@ -258,9 +258,31 @@ def handle(self, *args, **kwargs): pass +# List of unique non-subdomain prefixes (e.g., data-) from endpoint.hostPrefix in the botocore specs. +# Subdomain-prefixes (e.g., api.) work properly unless DNS rebind protection blocks DNS resolution, but +# these `-` dash-prefixes require special consideration. +# IMPORTANT: Adding a new host prefix here requires deploying a public DNS entry to ensure proper DNS resolution for +# such non-dot prefixed domains (e.g., data-localhost.localstack.cloud) +HOST_PREFIXES_NO_SUBDOMAIN = [ + "analytics-", + "control-storage-", + "data-", + "discovery-", + "query-", + "runtime-", + "storage-", + "streaming-", + "sync-", + "tags-", + "workflows-", +] +HOST_PREFIX_NAME_PATTERNS = [ + f"{host_prefix}{LOCALHOST_HOSTNAME}" for host_prefix in HOST_PREFIXES_NO_SUBDOMAIN +] + NAME_PATTERNS_POINTING_TO_LOCALSTACK = [ f".*{LOCALHOST_HOSTNAME}", -] +].extend(HOST_PREFIX_NAME_PATTERNS) def exclude_from_resolution(domain_regex: str): diff --git a/tests/unit/test_dns_server.py b/tests/unit/test_dns_server.py index 4e72b979c7389..1e3dbd8c1bf0e 100644 --- a/tests/unit/test_dns_server.py +++ b/tests/unit/test_dns_server.py @@ -5,8 +5,14 @@ import pytest from localstack import config +from localstack.aws.spec import iterate_service_operations from localstack.dns.models import AliasTarget, RecordType, SOARecord, TargetRecord -from localstack.dns.server import DnsServer, add_resolv_entry, get_fallback_dns_server +from localstack.dns.server import ( + HOST_PREFIXES_NO_SUBDOMAIN, + DnsServer, + add_resolv_entry, + get_fallback_dns_server, +) from localstack.utils.net import get_free_udp_port from localstack.utils.sync import retry @@ -460,3 +466,19 @@ def test_no_resolv_conf_overwriting_on_host(self, tmp_path: Path, monkeypatch): assert "nameserver 127.0.0.1" not in new_contents.splitlines() assert "nameserver 127.0.0.11" in new_contents.splitlines() + + def test_host_prefix_no_subdomain( + self, + ): + """This tests help to detect any potential future new host prefix domains added to the botocore specs. + If this test fails: + 1) Add the new entry to `HOST_PREFIXES_NO_SUBDOMAIN` to reflect any changes + 2) IMPORTANT: Add a public DNS entry for the given host prefix! + """ + unique_prefixes = set() + for service_model, operation in iterate_service_operations(): + if operation.endpoint and operation.endpoint.get("hostPrefix"): + unique_prefixes.add(operation.endpoint["hostPrefix"]) + + non_dot_unique_prefixes = [prefix for prefix in unique_prefixes if not prefix.endswith(".")] + assert set(HOST_PREFIXES_NO_SUBDOMAIN) == set(non_dot_unique_prefixes) From 17a7c147caf3613fa3adfcc8ff65a539ba9d32b4 Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Mon, 26 May 2025 15:39:20 +0200 Subject: [PATCH 2/6] Fix assignment error with extend returning None --- localstack-core/localstack/dns/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/localstack-core/localstack/dns/server.py b/localstack-core/localstack/dns/server.py index 0e708cfa71a38..736741dadf5fd 100644 --- a/localstack-core/localstack/dns/server.py +++ b/localstack-core/localstack/dns/server.py @@ -282,7 +282,8 @@ def handle(self, *args, **kwargs): NAME_PATTERNS_POINTING_TO_LOCALSTACK = [ f".*{LOCALHOST_HOSTNAME}", -].extend(HOST_PREFIX_NAME_PATTERNS) + *HOST_PREFIX_NAME_PATTERNS, +] def exclude_from_resolution(domain_regex: str): From 79b4573e7663447e1931e2bc0f3962afe3d02075 Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Mon, 26 May 2025 16:32:47 +0200 Subject: [PATCH 3/6] Add hostPrefix test cases for DNS resolution --- tests/bootstrap/test_dns_server.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/bootstrap/test_dns_server.py b/tests/bootstrap/test_dns_server.py index 32839b5a2944e..fbb31df4f3a83 100644 --- a/tests/bootstrap/test_dns_server.py +++ b/tests/bootstrap/test_dns_server.py @@ -149,14 +149,26 @@ def test_resolve_localstack_host( container_ip = running_container.ip_address() + # domain stdout, _ = dns_query_from_container(name=LOCALHOST_HOSTNAME, ip_address=container_ip) assert container_ip in stdout.decode().splitlines() + # domain with known hostPrefix (see test_host_prefix_no_subdomain) + stdout, _ = dns_query_from_container(name=f"data-{LOCALHOST_HOSTNAME}", ip_address=container_ip) + assert container_ip in stdout.decode().splitlines() + + # subdomain stdout, _ = dns_query_from_container(name=f"foo.{LOCALHOST_HOSTNAME}", ip_address=container_ip) assert container_ip in stdout.decode().splitlines() + # domain stdout, _ = dns_query_from_container(name=localstack_host, ip_address=container_ip) assert container_ip in stdout.decode().splitlines() + # domain with known hostPrefix (see test_host_prefix_no_subdomain) + stdout, _ = dns_query_from_container(name=f"data-{localstack_host}", ip_address=container_ip) + assert container_ip in stdout.decode().splitlines() + + # subdomain stdout, _ = dns_query_from_container(name=f"foo.{localstack_host}", ip_address=container_ip) assert container_ip in stdout.decode().splitlines() From 27cc63113f4a1fb9b1f15c06fd385850af59cb90 Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Mon, 26 May 2025 18:51:56 +0200 Subject: [PATCH 4/6] Add test cases for host prefix resolution within Lambda # Conflicts: # tests/aws/services/lambda_/test_lambda.snapshot.json --- localstack-core/localstack/dns/server.py | 1 + .../functions/host_prefix_operation.py | 71 +++++++++++++++++++ tests/aws/services/lambda_/test_lambda.py | 31 ++++++++ .../lambda_/test_lambda.snapshot.json | 24 +++++++ .../lambda_/test_lambda.validation.json | 3 + 5 files changed, 130 insertions(+) create mode 100644 tests/aws/services/lambda_/functions/host_prefix_operation.py diff --git a/localstack-core/localstack/dns/server.py b/localstack-core/localstack/dns/server.py index 736741dadf5fd..7516e391d4572 100644 --- a/localstack-core/localstack/dns/server.py +++ b/localstack-core/localstack/dns/server.py @@ -263,6 +263,7 @@ def handle(self, *args, **kwargs): # these `-` dash-prefixes require special consideration. # IMPORTANT: Adding a new host prefix here requires deploying a public DNS entry to ensure proper DNS resolution for # such non-dot prefixed domains (e.g., data-localhost.localstack.cloud) +# LIMITATION: As of 2025-05-26, only used prefixes are deployed to our public DNS, including `sync-` and `data-` HOST_PREFIXES_NO_SUBDOMAIN = [ "analytics-", "control-storage-", diff --git a/tests/aws/services/lambda_/functions/host_prefix_operation.py b/tests/aws/services/lambda_/functions/host_prefix_operation.py new file mode 100644 index 0000000000000..ccc49da725a62 --- /dev/null +++ b/tests/aws/services/lambda_/functions/host_prefix_operation.py @@ -0,0 +1,71 @@ +import json +import os +from urllib.parse import urlparse + +import boto3 +from botocore.config import Config + +region = os.environ["AWS_REGION"] +account = boto3.client("sts").get_caller_identity()["Account"] +state_machine_arn_doesnotexist = ( + f"arn:aws:states:{region}:{account}:stateMachine:doesNotExistStateMachine" +) + + +def do_test(test_case): + sfn_client = test_case["client"] + try: + sfn_client.start_sync_execution( + stateMachineArn=state_machine_arn_doesnotexist, + input=json.dumps({}), + name="SyncExecution", + ) + return {"status": "failure"} + except sfn_client.exceptions.StateMachineDoesNotExist: + # We are testing the error case here, so we expect this exception to be raised. + # Testing the error case simplifies the test case because we don't need to set up a StepFunction. + return {"status": "success"} + except Exception as e: + return {"status": "exception", "exception": str(e)} + + +def handler(event, context): + # The environment variable AWS_ENDPOINT_URL is only available in LocalStack + aws_endpoint_url = os.environ.get("AWS_ENDPOINT_URL") + + host_prefix_client = boto3.client( + "stepfunctions", + endpoint_url=os.environ.get("AWS_ENDPOINT_URL"), + ) + localstack_adjusted_domain = None + # The localstack domain only works in LocalStack, None is ignored + if aws_endpoint_url: + port = urlparse(aws_endpoint_url).port + localstack_adjusted_domain = f"http://localhost.localstack.cloud:{port}" + host_prefix_client_localstack_domain = boto3.client( + "stepfunctions", + endpoint_url=localstack_adjusted_domain, + ) + no_host_prefix_client = boto3.client( + "stepfunctions", + endpoint_url=os.environ.get("AWS_ENDPOINT_URL"), + config=Config(inject_host_prefix=False), + ) + + test_cases = [ + {"name": "host_prefix", "client": host_prefix_client}, + {"name": "host_prefix_localstack_domain", "client": host_prefix_client_localstack_domain}, + # Omitting the host prefix can only work in LocalStack + { + "name": "no_host_prefix", + "client": no_host_prefix_client if aws_endpoint_url else host_prefix_client, + }, + ] + + test_results = {} + for test_case in test_cases: + test_name = test_case["name"] + test_result = do_test(test_case) + test_results[test_name] = test_result + + return test_results diff --git a/tests/aws/services/lambda_/test_lambda.py b/tests/aws/services/lambda_/test_lambda.py index 7da990f63c716..08d8d77f0e4f2 100644 --- a/tests/aws/services/lambda_/test_lambda.py +++ b/tests/aws/services/lambda_/test_lambda.py @@ -129,6 +129,7 @@ TEST_LAMBDA_NOTIFIER = os.path.join(THIS_FOLDER, "functions/lambda_notifier.py") TEST_LAMBDA_CLOUDWATCH_LOGS = os.path.join(THIS_FOLDER, "functions/lambda_cloudwatch_logs.py") TEST_LAMBDA_XRAY_TRACEID = os.path.join(THIS_FOLDER, "functions/xray_tracing_traceid.py") +TEST_LAMBDA_HOST_PREFIX_OPERATION = os.path.join(THIS_FOLDER, "functions/host_prefix_operation.py") PYTHON_TEST_RUNTIMES = RUNTIMES_AGGREGATED["python"] NODE_TEST_RUNTIMES = RUNTIMES_AGGREGATED["nodejs"] @@ -830,6 +831,36 @@ def test_lambda_init_environment( result = aws_client.lambda_.invoke(FunctionName=func_name, Payload=json.dumps({"pid": 1})) snapshot.match("lambda-init-inspection", result) + @markers.aws.validated + @pytest.mark.skipif( + not config.use_custom_dns(), + reason="Host prefix cannot be resolved if DNS server is disabled", + ) + @markers.snapshot.skip_snapshot_verify( + paths=[ + # TODO: Fix hostPrefix operations failing by default within Lambda + # Idea: Support prefixed and non-prefixed operations by default and botocore should drop the prefix for + # non-supported hostnames such as IPv4 (e.g., `sync-192.168.65.254`) + "$..Payload.host_prefix.*", + ], + ) + def test_lambda_host_prefix_api_operation(self, create_lambda_function, aws_client, snapshot): + """Ensure that API operations with a hostPrefix are forwarded to the LocalStack instance. Examples: + * StartSyncExecution: https://docs.aws.amazon.com/step-functions/latest/apireference/API_StartSyncExecution.html + * DiscoverInstances: https://docs.aws.amazon.com/cloud-map/latest/api/API_DiscoverInstances.html + hostPrefix background test_host_prefix_no_subdomain + StepFunction example for the hostPrefix `sync-` based on test_start_sync_execution + """ + func_name = f"test_lambda_{short_uid()}" + create_lambda_function( + func_name=func_name, + handler_file=TEST_LAMBDA_HOST_PREFIX_OPERATION, + runtime=Runtime.python3_12, + ) + invoke_result = aws_client.lambda_.invoke(FunctionName=func_name) + assert "FunctionError" not in invoke_result + snapshot.match("invoke-result", invoke_result) + URL_HANDLER_CODE = """ def handler(event, ctx): diff --git a/tests/aws/services/lambda_/test_lambda.snapshot.json b/tests/aws/services/lambda_/test_lambda.snapshot.json index 5c50c87363539..121d9b01ef397 100644 --- a/tests/aws/services/lambda_/test_lambda.snapshot.json +++ b/tests/aws/services/lambda_/test_lambda.snapshot.json @@ -4580,5 +4580,29 @@ } } } + }, + "tests/aws/services/lambda_/test_lambda.py::TestLambdaBehavior::test_lambda_host_prefix_api_operation": { + "recorded-date": "26-05-2025, 16:38:54", + "recorded-content": { + "invoke-result": { + "ExecutedVersion": "$LATEST", + "Payload": { + "host_prefix": { + "status": "success" + }, + "host_prefix_localstack_domain": { + "status": "success" + }, + "no_host_prefix": { + "status": "success" + } + }, + "StatusCode": 200, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + } + } } } diff --git a/tests/aws/services/lambda_/test_lambda.validation.json b/tests/aws/services/lambda_/test_lambda.validation.json index c456b8486fcc0..9b5d816f5ac1e 100644 --- a/tests/aws/services/lambda_/test_lambda.validation.json +++ b/tests/aws/services/lambda_/test_lambda.validation.json @@ -32,6 +32,9 @@ "tests/aws/services/lambda_/test_lambda.py::TestLambdaBehavior::test_lambda_cache_local[python]": { "last_validated_date": "2024-04-08T16:55:59+00:00" }, + "tests/aws/services/lambda_/test_lambda.py::TestLambdaBehavior::test_lambda_host_prefix_api_operation": { + "last_validated_date": "2025-05-26T16:38:53+00:00" + }, "tests/aws/services/lambda_/test_lambda.py::TestLambdaBehavior::test_lambda_init_environment": { "last_validated_date": "2024-04-08T16:56:25+00:00" }, From e5ee939ddde33cbccbb408c158b875c2e8a6c229 Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Mon, 26 May 2025 20:13:12 +0200 Subject: [PATCH 5/6] Remove prefix discovery- removed in a recent botocore update Likely updated related to this change (just a guess): https://github.com/localstack/localstack/pull/12645/files#diff-1d20d60454c412d95e42d6c9d2626a4389b249eb1cc22a8c1a815cf81b1893f5L62 Couldn't find any operations that use the `discovery-` prefix anymore. ```py discovery_prefixes = df[df['hostPrefix'].str.startswith('discovery-', na=False)] ``` --- localstack-core/localstack/dns/server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/localstack-core/localstack/dns/server.py b/localstack-core/localstack/dns/server.py index 7516e391d4572..f32d81292c75e 100644 --- a/localstack-core/localstack/dns/server.py +++ b/localstack-core/localstack/dns/server.py @@ -268,7 +268,6 @@ def handle(self, *args, **kwargs): "analytics-", "control-storage-", "data-", - "discovery-", "query-", "runtime-", "storage-", From a9ec9bba3713b67ce841842c44622b6d988b7f37 Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Mon, 2 Jun 2025 11:41:53 +0200 Subject: [PATCH 6/6] Add real validation of NAME_PATTERNS_POINTING_TO_LOCALSTACK --- tests/unit/test_dns_server.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/unit/test_dns_server.py b/tests/unit/test_dns_server.py index 1e3dbd8c1bf0e..96ffd172b1f7a 100644 --- a/tests/unit/test_dns_server.py +++ b/tests/unit/test_dns_server.py @@ -6,9 +6,11 @@ from localstack import config from localstack.aws.spec import iterate_service_operations +from localstack.constants import LOCALHOST_HOSTNAME from localstack.dns.models import AliasTarget, RecordType, SOARecord, TargetRecord from localstack.dns.server import ( HOST_PREFIXES_NO_SUBDOMAIN, + NAME_PATTERNS_POINTING_TO_LOCALSTACK, DnsServer, add_resolv_entry, get_fallback_dns_server, @@ -481,4 +483,9 @@ def test_host_prefix_no_subdomain( unique_prefixes.add(operation.endpoint["hostPrefix"]) non_dot_unique_prefixes = [prefix for prefix in unique_prefixes if not prefix.endswith(".")] + # Intermediary validation to easily summarize all differences assert set(HOST_PREFIXES_NO_SUBDOMAIN) == set(non_dot_unique_prefixes) + + # Real validation of NAME_PATTERNS_POINTING_TO_LOCALSTACK + for host_prefix in non_dot_unique_prefixes: + assert f"{host_prefix}{LOCALHOST_HOSTNAME}" in NAME_PATTERNS_POINTING_TO_LOCALSTACK