localstack · alexrashed · May 16, 2023 · May 12, 2023 · May 12, 2023 · May 13, 2023
diff --git a/localstack/config.py b/localstack/config.py
@@ -1,7 +1,6 @@
 import logging
 import os
 import platform
-import re
 import socket
 import subprocess
 import tempfile
@@ -15,7 +14,6 @@
     DEFAULT_BUCKET_MARKER_LOCAL,
     DEFAULT_DEVELOP_PORT,
     DEFAULT_LAMBDA_CONTAINER_REGISTRY,
-    DEFAULT_SERVICE_PORTS,
     DEFAULT_VOLUME_DIR,
     ENV_INTERNAL_TEST_COLLECT_METRIC,
     ENV_INTERNAL_TEST_RUN,
@@ -1178,57 +1176,20 @@ def collect_config_items() -> List[Tuple[str, Any]]:
     return result
 
 
-def parse_service_ports() -> Dict[str, int]:
-    """Parses the environment variable $SERVICES with a comma-separated list of services
-    and (optional) ports they should run on: 'service1:port1,service2,service3:port3'"""
-    service_ports = os.environ.get("SERVICES", "").strip()
-    if service_ports and not is_env_true("EAGER_SERVICE_LOADING"):
-        LOG.warning("SERVICES variable is ignored if EAGER_SERVICE_LOADING=0.")
-        service_ports = None  # TODO remove logic once we clear up the service ports stuff
-    if not service_ports:
-        return DEFAULT_SERVICE_PORTS
-    result = {}
-    for service_port in re.split(r"\s*,\s*", service_ports):
-        parts = re.split(r"[:=]", service_port)
-        service = parts[0]
-        key_upper = service.upper().replace("-", "_")
-        port_env_name = "%s_PORT" % key_upper
-        # (1) set default port number
-        port_number = DEFAULT_SERVICE_PORTS.get(service)
-        # (2) set port number from <SERVICE>_PORT environment, if present
-        if os.environ.get(port_env_name):
-            port_number = os.environ.get(port_env_name)
-        # (3) set port number from <service>:<port> portion in $SERVICES, if present
-        if len(parts) > 1:
-            port_number = int(parts[-1])
-        # (4) try to parse as int, fall back to 0 (invalid port)
-        try:
-            port_number = int(port_number)
-        except Exception:
-            port_number = 0
-        result[service] = port_number
-    return result
-
-
-# TODO: use functools cache, instead of global variable here
-SERVICE_PORTS = parse_service_ports()
-
-
 def populate_config_env_var_names():
     global CONFIG_ENV_VARS
 
-    for key, value in DEFAULT_SERVICE_PORTS.items():
-        clean_key = key.upper().replace("-", "_")
-        CONFIG_ENV_VARS += [
-            clean_key + "_BACKEND",
-            clean_key + "_PORT_EXTERNAL",
-            "PROVIDER_OVERRIDE_" + clean_key,
-        ]
+    CONFIG_ENV_VARS += [
+        key
+        for key in [key.upper() for key in os.environ]
+        if key.startswith("LOCALSTACK_") or key.startswith("PROVIDER_OVERRIDE_")
+    ]
 
     # create variable aliases prefixed with LOCALSTACK_ (except LOCALSTACK_HOSTNAME)
     CONFIG_ENV_VARS += [
         "LOCALSTACK_" + v for v in CONFIG_ENV_VARS if not v.startswith("LOCALSTACK_")
     ]
+
     CONFIG_ENV_VARS = list(set(CONFIG_ENV_VARS))
 
 
@@ -1241,14 +1202,7 @@ def service_port(service_key: str, external: bool = False) -> int:
     if external:
         if service_key == "sqs" and SQS_PORT_EXTERNAL:
             return SQS_PORT_EXTERNAL
-    if FORWARD_EDGE_INMEM:
-        if service_key == "elasticsearch":
-            # TODO Elasticsearch domains are a special case - we do not want to route them through
-            #  the edge service, as that would require too many route mappings. In the future, we
-            #  should integrate them with the port range for external services (4510-4530)
-            return SERVICE_PORTS.get(service_key, 0)
-        return get_edge_port_http()
-    return SERVICE_PORTS.get(service_key, 0)
+    return get_edge_port_http()
 
 
 def get_protocol():

diff --git a/localstack/constants.py b/localstack/constants.py
@@ -1,7 +1,5 @@
 import os
 
-import localstack_client.config
-
 import localstack
 
 # LocalStack version
@@ -40,9 +38,6 @@
 SSL_CERT_URL = f"{ARTIFACTS_REPO}/raw/master/local-certs/server.key"
 SSL_CERT_URL_FALLBACK = "{api_endpoint}/proxy/localstack.cert.key"
 
-# map of default service APIs and ports to be spun up (fetch map from localstack_client)
-DEFAULT_SERVICE_PORTS = localstack_client.config.get_service_ports()
-
 # host to bind to when starting the services
 BIND_HOST = "0.0.0.0"
 
@@ -194,8 +189,7 @@
 # list of official docker images
 OFFICIAL_IMAGES = [
     "localstack/localstack",
-    "localstack/localstack-light",
-    "localstack/localstack-full",
+    "localstack/localstack-pro",
 ]
 
 # s3 virtual host name

diff --git a/localstack/utils/bootstrap.py b/localstack/utils/bootstrap.py
@@ -10,6 +10,7 @@
 from typing import Dict, Iterable, List, Optional, Set
 
 from localstack import config, constants
+from localstack.config import get_edge_port_http, is_env_true
 from localstack.constants import DEFAULT_VOLUME_DIR
 from localstack.runtime import hooks
 from localstack.utils.container_networking import get_main_container_name
@@ -213,7 +214,26 @@ def get_enabled_apis() -> Set[str]:
 
     The result is cached, so it's safe to call. Clear the cache with get_enabled_apis.cache_clear().
     """
-    return resolve_apis(config.parse_service_ports().keys())
+    services_env = os.environ.get("SERVICES", "").strip()
+    services = None
+    if services_env and not is_env_true("EAGER_SERVICE_LOADING"):
+        LOG.warning("SERVICES variable is ignored if EAGER_SERVICE_LOADING=0.")
+    elif services_env:
+        # SERVICES and EAGER_SERVICE_LOADING are set
+        # SERVICES env var might contain ports, but we do not support these anymore
+        services = []
+        for service_port in re.split(r"\s*,\s*", services_env):
+            # Only extract the service name, discard the port
+            parts = re.split(r"[:=]", service_port)
+            service = parts[0]
+            services.append(service)
+
+    if not services:
+        from localstack.services.plugins import SERVICE_PLUGINS
+
+        services = SERVICE_PLUGINS.list_available()
+
+    return resolve_apis(services)
 
 
 # DEPRECATED, lazy loading should be assumed
@@ -510,12 +530,7 @@ def configure_container(container: LocalstackContainer):
     hooks.configure_localstack_container.run(container)
 
     # construct default port mappings
-    service_ports = config.SERVICE_PORTS
-    if service_ports.get("edge") == 0:
-        service_ports.pop("edge")
-    for port in service_ports.values():
-        if port:
-            container.ports.add(port)
+    container.ports.add(get_edge_port_http())
     for port in range(config.EXTERNAL_SERVICE_PORTS_START, config.EXTERNAL_SERVICE_PORTS_END):
         container.ports.add(port)
 

diff --git a/localstack/utils/diagnose.py b/localstack/utils/diagnose.py
@@ -46,7 +46,6 @@
 
 EXCLUDE_CONFIG_KEYS = {
     "CONFIG_ENV_VARS",
-    "DEFAULT_SERVICE_PORTS",
     "copyright",
     "__builtins__",
     "__cached__",

diff --git a/localstack/utils/strings.py b/localstack/utils/strings.py
@@ -9,8 +9,6 @@
 import zlib
 from typing import Dict, List, Union
 
-from botocore.httpchecksum import CrtCrc32cChecksum
-
 from localstack.config import DEFAULT_ENCODING
 
 _unprintables = (
@@ -153,6 +151,9 @@ def checksum_crc32(string: Union[str, bytes]) -> str:
 
 
 def checksum_crc32c(string: Union[str, bytes]):
+    # import botocore locally here to avoid a dependency of the CLI to botocore
+    from botocore.httpchecksum import CrtCrc32cChecksum
+
     checksum = CrtCrc32cChecksum()
     checksum.update(to_bytes(string))
     return base64.b64encode(checksum.digest()).decode()

diff --git a/setup.cfg b/setup.cfg
@@ -24,13 +24,11 @@ packages=find:
 
 # dependencies that are required for the cli (via pip install localstack)
 install_requires =
-    boto3>=1.26.121
     click>=7.0
     cachetools~=5.0.0
     cryptography
     dill==0.3.2
     dnspython>=1.16.0
-    localstack-client>=2.0
     plux>=1.3.1
     psutil>=5.4.8,<6.0.0
     python-dotenv>=0.19.1
@@ -44,7 +42,6 @@ install_requires =
     # needed for python3.7 compat (TypedDict, Literal, type hints)
     typing-extensions; python_version < '3.8'
     tailer>=0.4.1
-    apispec>=5.1.1
 
 [options.packages.find]
 exclude =
@@ -64,13 +61,14 @@ localstack =
 # required to actually run localstack on the host
 runtime =
     airspeed-ext==0.5.19
-    # TODO: check amazon_kclpy pin once build failure in 2.1.0 has been fixed
-    amazon_kclpy>=2.0.6,<2.1.0
+    amazon_kclpy>=2.0.6,!=2.1.0
     antlr4-python3-runtime==4.11.1
+    apispec>=5.1.1
     aws-sam-translator>=1.15.1
     awscli>=1.22.90
     awscrt>=0.13.14
     boto>=2.49.0
+    boto3>=1.26.121
     botocore>=1.29.121,<=1.29.121
     cbor2>=5.2.0
     crontab>=0.22.6
@@ -85,6 +83,7 @@ runtime =
     jsonpatch>=1.24,<2.0
     jsonpath-ng>=1.5.3
     jsonpath-rw>=1.4.0,<2.0.0
+    localstack-client>=2.0
     moto-ext[all]==4.1.8.post1
     opensearch-py==2.1.1
     pproxy>=2.7.0

diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py
@@ -1,23 +1,8 @@
-import os
-from contextlib import contextmanager
-from typing import Any, Dict
-
 import pytest
 
 from localstack import config
 
 
-@contextmanager
-def temporary_env(env: Dict[str, Any]):
-    old = os.environ.copy()
-    try:
-        os.environ.update(env)
-        yield os.environ
-    finally:
-        os.environ.clear()
-        os.environ.update(old)
-
-
 class TestProviderConfig:
     def test_provider_default_value(self):
         default_value = "default_value"
@@ -73,65 +58,6 @@ def test_bulk_set_if_not_exists(self):
         assert provider_config.get_provider("kinesis") == default_value
 
 
-class TestParseServicePorts:
-    def test_returns_default_service_ports(self):
-        result = config.parse_service_ports()
-        assert result == config.DEFAULT_SERVICE_PORTS
-
-    def test_with_service_subset(self):
-        with temporary_env({"SERVICES": "s3,sqs", "EAGER_SERVICE_LOADING": "1"}):
-            result = config.parse_service_ports()
-
-        assert len(result) == 2
-        assert "s3" in result
-        assert "sqs" in result
-        assert result["s3"] == 4566
-        assert result["sqs"] == 4566
-
-    def test_custom_service_default_port(self):
-        with temporary_env({"SERVICES": "foobar", "EAGER_SERVICE_LOADING": "1"}):
-            result = config.parse_service_ports()
-
-        assert len(result) == 1
-        assert "foobar" not in config.DEFAULT_SERVICE_PORTS
-        assert "foobar" in result
-        # foobar is not a default service so it is assigned 0
-        assert result["foobar"] == 0
-
-    def test_custom_port_mapping(self):
-        with temporary_env(
-            {"SERVICES": "foobar", "FOOBAR_PORT": "1234", "EAGER_SERVICE_LOADING": "1"}
-        ):
-            result = config.parse_service_ports()
-
-        assert len(result) == 1
-        assert "foobar" not in config.DEFAULT_SERVICE_PORTS
-        assert "foobar" in result
-        assert result["foobar"] == 1234
-
-    def test_custom_illegal_port_mapping(self):
-        with temporary_env(
-            {"SERVICES": "foobar", "FOOBAR_PORT": "asdf", "EAGER_SERVICE_LOADING": "1"}
-        ):
-            result = config.parse_service_ports()
-
-        assert len(result) == 1
-        assert "foobar" not in config.DEFAULT_SERVICE_PORTS
-        assert "foobar" in result
-        # FOOBAR_PORT cannot be parsed
-        assert result["foobar"] == 0
-
-    def test_custom_port_mapping_in_services_env(self):
-        with temporary_env({"SERVICES": "foobar:1235", "EAGER_SERVICE_LOADING": "1"}):
-            result = config.parse_service_ports()
-
-        assert len(result) == 1
-        assert "foobar" not in config.DEFAULT_SERVICE_PORTS
-        assert "foobar" in result
-        # FOOBAR_PORT cannot be parsed
-        assert result["foobar"] == 1235
-
-
 class TestEdgeVariablesDerivedCorrectly:
     """
     Post-v2 we are deriving