From 5e9656dea41447298c19c19ee9041e4aa0168020 Mon Sep 17 00:00:00 2001 From: Alexander Rashed Date: Fri, 12 May 2023 11:30:57 +0200 Subject: [PATCH 1/5] remove install dependency on localstack_client --- localstack/config.py | 89 ++++++++++++++++------------------- localstack/constants.py | 8 +--- localstack/utils/bootstrap.py | 16 ++++--- localstack/utils/diagnose.py | 1 - setup.cfg | 9 ++-- tests/unit/test_config.py | 59 ----------------------- 6 files changed, 55 insertions(+), 127 deletions(-) diff --git a/localstack/config.py b/localstack/config.py index cb7fdcd39622f..e4934635e33eb 100644 --- a/localstack/config.py +++ b/localstack/config.py @@ -1,7 +1,6 @@ import logging import os import platform -import re import socket import subprocess import tempfile @@ -15,7 +14,6 @@ DEFAULT_BUCKET_MARKER_LOCAL, DEFAULT_DEVELOP_PORT, DEFAULT_LAMBDA_CONTAINER_REGISTRY, - DEFAULT_SERVICE_PORTS, DEFAULT_VOLUME_DIR, ENV_INTERNAL_TEST_COLLECT_METRIC, ENV_INTERNAL_TEST_RUN, @@ -1016,6 +1014,44 @@ def get_gateway_listen(gateway_listen: str) -> List[HostAndPort]: # Whether to return and parse access key ids starting with an "A", like on AWS PARITY_AWS_ACCESS_KEY_ID = is_env_true("PARITY_AWS_ACCESS_KEY_ID") +# List of services supported by LocalStack +SUPPORTED_SERVICES = [ + "acm", + "apigateway", + "cloudformation", + "cloudwatch", + "config", + "dynamodb", + "dynamodbstreams", + "ec2", + "es", + "events", + "firehose", + "iam", + "kinesis", + "kms", + "lambda", + "logs", + "opensearch", + "redshift", + "resource-groups", + "resourcegroupstaggingapi", + "route53", + "route53resolver", + "s3", + "s3control", + "secretsmanager", + "ses", + "sns", + "sqs", + "ssm", + "stepfunctions", + "sts", + "support", + "swf", + "transcribe", +] + # HINT: Please add deprecated environment variables to deprecations.py # list of environment variable names used for configuration. @@ -1178,47 +1214,11 @@ def collect_config_items() -> List[Tuple[str, Any]]: return result -def parse_service_ports() -> Dict[str, int]: - """Parses the environment variable $SERVICES with a comma-separated list of services - and (optional) ports they should run on: 'service1:port1,service2,service3:port3'""" - service_ports = os.environ.get("SERVICES", "").strip() - if service_ports and not is_env_true("EAGER_SERVICE_LOADING"): - LOG.warning("SERVICES variable is ignored if EAGER_SERVICE_LOADING=0.") - service_ports = None # TODO remove logic once we clear up the service ports stuff - if not service_ports: - return DEFAULT_SERVICE_PORTS - result = {} - for service_port in re.split(r"\s*,\s*", service_ports): - parts = re.split(r"[:=]", service_port) - service = parts[0] - key_upper = service.upper().replace("-", "_") - port_env_name = "%s_PORT" % key_upper - # (1) set default port number - port_number = DEFAULT_SERVICE_PORTS.get(service) - # (2) set port number from _PORT environment, if present - if os.environ.get(port_env_name): - port_number = os.environ.get(port_env_name) - # (3) set port number from : portion in $SERVICES, if present - if len(parts) > 1: - port_number = int(parts[-1]) - # (4) try to parse as int, fall back to 0 (invalid port) - try: - port_number = int(port_number) - except Exception: - port_number = 0 - result[service] = port_number - return result - - -# TODO: use functools cache, instead of global variable here -SERVICE_PORTS = parse_service_ports() - - def populate_config_env_var_names(): global CONFIG_ENV_VARS - for key, value in DEFAULT_SERVICE_PORTS.items(): - clean_key = key.upper().replace("-", "_") + for service in SUPPORTED_SERVICES: + clean_key = service.upper().replace("-", "_") CONFIG_ENV_VARS += [ clean_key + "_BACKEND", clean_key + "_PORT_EXTERNAL", @@ -1241,14 +1241,7 @@ def service_port(service_key: str, external: bool = False) -> int: if external: if service_key == "sqs" and SQS_PORT_EXTERNAL: return SQS_PORT_EXTERNAL - if FORWARD_EDGE_INMEM: - if service_key == "elasticsearch": - # TODO Elasticsearch domains are a special case - we do not want to route them through - # the edge service, as that would require too many route mappings. In the future, we - # should integrate them with the port range for external services (4510-4530) - return SERVICE_PORTS.get(service_key, 0) - return get_edge_port_http() - return SERVICE_PORTS.get(service_key, 0) + return get_edge_port_http() def get_protocol(): diff --git a/localstack/constants.py b/localstack/constants.py index c103c24593b63..ff58909899230 100644 --- a/localstack/constants.py +++ b/localstack/constants.py @@ -1,7 +1,5 @@ import os -import localstack_client.config - import localstack # LocalStack version @@ -40,9 +38,6 @@ SSL_CERT_URL = f"{ARTIFACTS_REPO}/raw/master/local-certs/server.key" SSL_CERT_URL_FALLBACK = "{api_endpoint}/proxy/localstack.cert.key" -# map of default service APIs and ports to be spun up (fetch map from localstack_client) -DEFAULT_SERVICE_PORTS = localstack_client.config.get_service_ports() - # host to bind to when starting the services BIND_HOST = "0.0.0.0" @@ -194,8 +189,7 @@ # list of official docker images OFFICIAL_IMAGES = [ "localstack/localstack", - "localstack/localstack-light", - "localstack/localstack-full", + "localstack/localstack-pro", ] # s3 virtual host name diff --git a/localstack/utils/bootstrap.py b/localstack/utils/bootstrap.py index b889ebbea4f13..1cc4908533e3b 100644 --- a/localstack/utils/bootstrap.py +++ b/localstack/utils/bootstrap.py @@ -10,6 +10,7 @@ from typing import Dict, Iterable, List, Optional, Set from localstack import config, constants +from localstack.config import SUPPORTED_SERVICES, get_edge_port_http, is_env_true from localstack.constants import DEFAULT_VOLUME_DIR from localstack.runtime import hooks from localstack.utils.container_networking import get_main_container_name @@ -213,7 +214,13 @@ def get_enabled_apis() -> Set[str]: The result is cached, so it's safe to call. Clear the cache with get_enabled_apis.cache_clear(). """ - return resolve_apis(config.parse_service_ports().keys()) + services = os.environ.get("SERVICES", "").strip() + if services and not is_env_true("EAGER_SERVICE_LOADING"): + LOG.warning("SERVICES variable is ignored if EAGER_SERVICE_LOADING=0.") + services = None + if not services: + services = SUPPORTED_SERVICES + return resolve_apis(services) # DEPRECATED, lazy loading should be assumed @@ -510,12 +517,7 @@ def configure_container(container: LocalstackContainer): hooks.configure_localstack_container.run(container) # construct default port mappings - service_ports = config.SERVICE_PORTS - if service_ports.get("edge") == 0: - service_ports.pop("edge") - for port in service_ports.values(): - if port: - container.ports.add(port) + container.ports.add(get_edge_port_http()) for port in range(config.EXTERNAL_SERVICE_PORTS_START, config.EXTERNAL_SERVICE_PORTS_END): container.ports.add(port) diff --git a/localstack/utils/diagnose.py b/localstack/utils/diagnose.py index 9f8413bf8a46b..d414602090f75 100644 --- a/localstack/utils/diagnose.py +++ b/localstack/utils/diagnose.py @@ -46,7 +46,6 @@ EXCLUDE_CONFIG_KEYS = { "CONFIG_ENV_VARS", - "DEFAULT_SERVICE_PORTS", "copyright", "__builtins__", "__cached__", diff --git a/setup.cfg b/setup.cfg index 3b45a3839f90c..498a344afc8d5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,13 +24,11 @@ packages=find: # dependencies that are required for the cli (via pip install localstack) install_requires = - boto3>=1.26.121 click>=7.0 cachetools~=5.0.0 cryptography dill==0.3.2 dnspython>=1.16.0 - localstack-client>=2.0 plux>=1.3.1 psutil>=5.4.8,<6.0.0 python-dotenv>=0.19.1 @@ -44,7 +42,6 @@ install_requires = # needed for python3.7 compat (TypedDict, Literal, type hints) typing-extensions; python_version < '3.8' tailer>=0.4.1 - apispec>=5.1.1 [options.packages.find] exclude = @@ -64,13 +61,14 @@ localstack = # required to actually run localstack on the host runtime = airspeed-ext==0.5.19 - # TODO: check amazon_kclpy pin once build failure in 2.1.0 has been fixed - amazon_kclpy>=2.0.6,<2.1.0 + amazon_kclpy>=2.0.6,!=2.1.0 antlr4-python3-runtime==4.11.1 + apispec>=5.1.1 aws-sam-translator>=1.15.1 awscli>=1.22.90 awscrt>=0.13.14 boto>=2.49.0 + boto3>=1.26.121 botocore>=1.29.121,<=1.29.121 cbor2>=5.2.0 crontab>=0.22.6 @@ -85,6 +83,7 @@ runtime = jsonpatch>=1.24,<2.0 jsonpath-ng>=1.5.3 jsonpath-rw>=1.4.0,<2.0.0 + localstack-client>=2.0 moto-ext[all]==4.1.8.post1 opensearch-py==2.1.1 pproxy>=2.7.0 diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index f2e875727ba98..64102467fcd21 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -73,65 +73,6 @@ def test_bulk_set_if_not_exists(self): assert provider_config.get_provider("kinesis") == default_value -class TestParseServicePorts: - def test_returns_default_service_ports(self): - result = config.parse_service_ports() - assert result == config.DEFAULT_SERVICE_PORTS - - def test_with_service_subset(self): - with temporary_env({"SERVICES": "s3,sqs", "EAGER_SERVICE_LOADING": "1"}): - result = config.parse_service_ports() - - assert len(result) == 2 - assert "s3" in result - assert "sqs" in result - assert result["s3"] == 4566 - assert result["sqs"] == 4566 - - def test_custom_service_default_port(self): - with temporary_env({"SERVICES": "foobar", "EAGER_SERVICE_LOADING": "1"}): - result = config.parse_service_ports() - - assert len(result) == 1 - assert "foobar" not in config.DEFAULT_SERVICE_PORTS - assert "foobar" in result - # foobar is not a default service so it is assigned 0 - assert result["foobar"] == 0 - - def test_custom_port_mapping(self): - with temporary_env( - {"SERVICES": "foobar", "FOOBAR_PORT": "1234", "EAGER_SERVICE_LOADING": "1"} - ): - result = config.parse_service_ports() - - assert len(result) == 1 - assert "foobar" not in config.DEFAULT_SERVICE_PORTS - assert "foobar" in result - assert result["foobar"] == 1234 - - def test_custom_illegal_port_mapping(self): - with temporary_env( - {"SERVICES": "foobar", "FOOBAR_PORT": "asdf", "EAGER_SERVICE_LOADING": "1"} - ): - result = config.parse_service_ports() - - assert len(result) == 1 - assert "foobar" not in config.DEFAULT_SERVICE_PORTS - assert "foobar" in result - # FOOBAR_PORT cannot be parsed - assert result["foobar"] == 0 - - def test_custom_port_mapping_in_services_env(self): - with temporary_env({"SERVICES": "foobar:1235", "EAGER_SERVICE_LOADING": "1"}): - result = config.parse_service_ports() - - assert len(result) == 1 - assert "foobar" not in config.DEFAULT_SERVICE_PORTS - assert "foobar" in result - # FOOBAR_PORT cannot be parsed - assert result["foobar"] == 1235 - - class TestEdgeVariablesDerivedCorrectly: """ Post-v2 we are deriving From 013bccdf626d1b099683d1d6e7d0dbf128e54c8c Mon Sep 17 00:00:00 2001 From: Alexander Rashed Date: Fri, 12 May 2023 11:51:08 +0200 Subject: [PATCH 2/5] fix dependency of string utils to botocore --- localstack/utils/strings.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/localstack/utils/strings.py b/localstack/utils/strings.py index 1305beb46b615..c287222569f15 100644 --- a/localstack/utils/strings.py +++ b/localstack/utils/strings.py @@ -9,8 +9,6 @@ import zlib from typing import Dict, List, Union -from botocore.httpchecksum import CrtCrc32cChecksum - from localstack.config import DEFAULT_ENCODING _unprintables = ( @@ -153,6 +151,9 @@ def checksum_crc32(string: Union[str, bytes]) -> str: def checksum_crc32c(string: Union[str, bytes]): + # import botocore locally here to avoid a dependency of the CLI to botocore + from botocore.httpchecksum import CrtCrc32cChecksum + checksum = CrtCrc32cChecksum() checksum.update(to_bytes(string)) return base64.b64encode(checksum.digest()).decode() From 08bf515ee93ce1052323eed7cebc56195c328558 Mon Sep 17 00:00:00 2001 From: Thomas Rausch Date: Sat, 13 May 2023 16:19:38 +0200 Subject: [PATCH 3/5] remove list of supported services --- localstack/config.py | 57 +++++------------------------------ localstack/utils/bootstrap.py | 6 ++-- 2 files changed, 11 insertions(+), 52 deletions(-) diff --git a/localstack/config.py b/localstack/config.py index e4934635e33eb..168f1d3b605fd 100644 --- a/localstack/config.py +++ b/localstack/config.py @@ -1014,44 +1014,6 @@ def get_gateway_listen(gateway_listen: str) -> List[HostAndPort]: # Whether to return and parse access key ids starting with an "A", like on AWS PARITY_AWS_ACCESS_KEY_ID = is_env_true("PARITY_AWS_ACCESS_KEY_ID") -# List of services supported by LocalStack -SUPPORTED_SERVICES = [ - "acm", - "apigateway", - "cloudformation", - "cloudwatch", - "config", - "dynamodb", - "dynamodbstreams", - "ec2", - "es", - "events", - "firehose", - "iam", - "kinesis", - "kms", - "lambda", - "logs", - "opensearch", - "redshift", - "resource-groups", - "resourcegroupstaggingapi", - "route53", - "route53resolver", - "s3", - "s3control", - "secretsmanager", - "ses", - "sns", - "sqs", - "ssm", - "stepfunctions", - "sts", - "support", - "swf", - "transcribe", -] - # HINT: Please add deprecated environment variables to deprecations.py # list of environment variable names used for configuration. @@ -1217,19 +1179,14 @@ def collect_config_items() -> List[Tuple[str, Any]]: def populate_config_env_var_names(): global CONFIG_ENV_VARS - for service in SUPPORTED_SERVICES: - clean_key = service.upper().replace("-", "_") - CONFIG_ENV_VARS += [ - clean_key + "_BACKEND", - clean_key + "_PORT_EXTERNAL", - "PROVIDER_OVERRIDE_" + clean_key, - ] - - # create variable aliases prefixed with LOCALSTACK_ (except LOCALSTACK_HOSTNAME) - CONFIG_ENV_VARS += [ - "LOCALSTACK_" + v for v in CONFIG_ENV_VARS if not v.startswith("LOCALSTACK_") + CONFIG_ENV_VARS = [ + key + for key in [key.upper() for key in os.environ] + if key.startswith("LOCALSTACK_") + if key.endswith("_BACKEND") + or key.endswith("_PORT_EXTERNAL") + or key.startswith("PROVIDER_OVERRIDE_") ] - CONFIG_ENV_VARS = list(set(CONFIG_ENV_VARS)) # populate env var names to be passed to the container diff --git a/localstack/utils/bootstrap.py b/localstack/utils/bootstrap.py index 1cc4908533e3b..090cb9e8b29e5 100644 --- a/localstack/utils/bootstrap.py +++ b/localstack/utils/bootstrap.py @@ -10,7 +10,7 @@ from typing import Dict, Iterable, List, Optional, Set from localstack import config, constants -from localstack.config import SUPPORTED_SERVICES, get_edge_port_http, is_env_true +from localstack.config import get_edge_port_http, is_env_true from localstack.constants import DEFAULT_VOLUME_DIR from localstack.runtime import hooks from localstack.utils.container_networking import get_main_container_name @@ -219,7 +219,9 @@ def get_enabled_apis() -> Set[str]: LOG.warning("SERVICES variable is ignored if EAGER_SERVICE_LOADING=0.") services = None if not services: - services = SUPPORTED_SERVICES + from localstack.services.plugins import SERVICE_PLUGINS + + services = SERVICE_PLUGINS.list_available() return resolve_apis(services) From 41c12d7e81a053cbd4731b4e7fb2da57582619b7 Mon Sep 17 00:00:00 2001 From: Alexander Rashed Date: Mon, 15 May 2023 15:42:24 +0200 Subject: [PATCH 4/5] re-add parsing service ports, re-add unit tests --- localstack/utils/bootstrap.py | 17 +++++-- tests/unit/test_config.py | 15 ------ tests/unit/utils/test_bootstrap.py | 73 ++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 18 deletions(-) create mode 100644 tests/unit/utils/test_bootstrap.py diff --git a/localstack/utils/bootstrap.py b/localstack/utils/bootstrap.py index 090cb9e8b29e5..ff6280ec5c03b 100644 --- a/localstack/utils/bootstrap.py +++ b/localstack/utils/bootstrap.py @@ -214,14 +214,25 @@ def get_enabled_apis() -> Set[str]: The result is cached, so it's safe to call. Clear the cache with get_enabled_apis.cache_clear(). """ - services = os.environ.get("SERVICES", "").strip() - if services and not is_env_true("EAGER_SERVICE_LOADING"): + services_env = os.environ.get("SERVICES", "").strip() + services = None + if services_env and not is_env_true("EAGER_SERVICE_LOADING"): LOG.warning("SERVICES variable is ignored if EAGER_SERVICE_LOADING=0.") - services = None + elif services_env: + # SERVICES and EAGER_SERVICE_LOADING are set + # SERVICES env var might contain ports, but we do not support these anymore + services = [] + for service_port in re.split(r"\s*,\s*", services_env): + # Only extract the service name, discard the port + parts = re.split(r"[:=]", service_port) + service = parts[0] + services.append(service) + if not services: from localstack.services.plugins import SERVICE_PLUGINS services = SERVICE_PLUGINS.list_available() + return resolve_apis(services) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 64102467fcd21..9b15880ef30fa 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -1,23 +1,8 @@ -import os -from contextlib import contextmanager -from typing import Any, Dict - import pytest from localstack import config -@contextmanager -def temporary_env(env: Dict[str, Any]): - old = os.environ.copy() - try: - os.environ.update(env) - yield os.environ - finally: - os.environ.clear() - os.environ.update(old) - - class TestProviderConfig: def test_provider_default_value(self): default_value = "default_value" diff --git a/tests/unit/utils/test_bootstrap.py b/tests/unit/utils/test_bootstrap.py new file mode 100644 index 0000000000000..3e14c23f8757a --- /dev/null +++ b/tests/unit/utils/test_bootstrap.py @@ -0,0 +1,73 @@ +import os +from contextlib import contextmanager +from typing import Any, Dict + +import pytest + +from localstack.utils.bootstrap import get_enabled_apis + + +@contextmanager +def temporary_env(env: Dict[str, Any]): + old = os.environ.copy() + try: + os.environ.update(env) + yield os.environ + finally: + os.environ.clear() + os.environ.update(old) + + +class TestGetEnabledServices: + @pytest.fixture(autouse=True) + def reset_get_enabled_apis(self): + """ + Ensures that the cache is reset on get_enabled_apis. + :return: get_enabled_apis method with reset fixture + """ + get_enabled_apis.cache_clear() + yield + get_enabled_apis.cache_clear() + + def test_returns_default_service_ports(self): + from localstack.services.plugins import SERVICE_PLUGINS + + result = get_enabled_apis() + assert result == set(SERVICE_PLUGINS.list_available()) + + def test_with_service_subset(self): + with temporary_env({"SERVICES": "s3,sqs", "EAGER_SERVICE_LOADING": "1"}): + result = get_enabled_apis() + + assert len(result) == 2 + assert "s3" in result + assert "sqs" in result + + def test_custom_service_without_port(self): + with temporary_env({"SERVICES": "foobar", "EAGER_SERVICE_LOADING": "1"}): + result = get_enabled_apis() + + assert len(result) == 1 + assert "foobar" in result + + def test_custom_port_mapping_in_services_env(self): + with temporary_env({"SERVICES": "foobar:1235", "EAGER_SERVICE_LOADING": "1"}): + result = get_enabled_apis() + + assert len(result) == 1 + assert "foobar" in result + + def test_resolve_meta(self): + with temporary_env({"SERVICES": "es,cognito:1337", "EAGER_SERVICE_LOADING": "1"}): + result = get_enabled_apis() + + assert len(result) == 4 + assert result == { + # directly given + "es", + # a dependency of es + "opensearch", + # "cognito" is a composite for "cognito-idp" and "cognito-identity" + "cognito-idp", + "cognito-identity", + } From 71a5b7c7b8a0736f0c5a1158213ab2db05ce828b Mon Sep 17 00:00:00 2001 From: Alexander Rashed Date: Mon, 15 May 2023 16:23:46 +0200 Subject: [PATCH 5/5] fix config var population --- localstack/config.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/localstack/config.py b/localstack/config.py index 168f1d3b605fd..82106b19ba320 100644 --- a/localstack/config.py +++ b/localstack/config.py @@ -1179,15 +1179,19 @@ def collect_config_items() -> List[Tuple[str, Any]]: def populate_config_env_var_names(): global CONFIG_ENV_VARS - CONFIG_ENV_VARS = [ + CONFIG_ENV_VARS += [ key for key in [key.upper() for key in os.environ] - if key.startswith("LOCALSTACK_") - if key.endswith("_BACKEND") - or key.endswith("_PORT_EXTERNAL") - or key.startswith("PROVIDER_OVERRIDE_") + if key.startswith("LOCALSTACK_") or key.startswith("PROVIDER_OVERRIDE_") ] + # create variable aliases prefixed with LOCALSTACK_ (except LOCALSTACK_HOSTNAME) + CONFIG_ENV_VARS += [ + "LOCALSTACK_" + v for v in CONFIG_ENV_VARS if not v.startswith("LOCALSTACK_") + ] + + CONFIG_ENV_VARS = list(set(CONFIG_ENV_VARS)) + # populate env var names to be passed to the container populate_config_env_var_names()