diff --git a/localstack-core/localstack/config.py b/localstack-core/localstack/config.py index 89583165b8787..b46b86e7ee92f 100644 --- a/localstack-core/localstack/config.py +++ b/localstack-core/localstack/config.py @@ -896,6 +896,20 @@ def populate_edge_configuration( # randomly inject faults to Kinesis KINESIS_ERROR_PROBABILITY = float(os.environ.get("KINESIS_ERROR_PROBABILITY", "").strip() or 0.0) +# SEMI-PUBLIC: "node" (default); not actively communicated +# Select whether to use the node or scala build when running Kinesis Mock +KINESIS_MOCK_PROVIDER_ENGINE = os.environ.get("KINESIS_MOCK_PROVIDER_ENGINE", "").strip() or "node" + +# set the maximum Java heap size corresponding to the '-Xmx' flag +KINESIS_MOCK_MAXIMUM_HEAP_SIZE = ( + os.environ.get("KINESIS_MOCK_MAXIMUM_HEAP_SIZE", "").strip() or "512m" +) + +# set the initial Java heap size corresponding to the '-Xms' flag +KINESIS_MOCK_INITIAL_HEAP_SIZE = ( + os.environ.get("KINESIS_MOCK_INITIAL_HEAP_SIZE", "").strip() or "256m" +) + # randomly inject faults to DynamoDB DYNAMODB_ERROR_PROBABILITY = float(os.environ.get("DYNAMODB_ERROR_PROBABILITY", "").strip() or 0.0) DYNAMODB_READ_ERROR_PROBABILITY = float( diff --git a/localstack-core/localstack/services/kinesis/kinesis_mock_server.py b/localstack-core/localstack/services/kinesis/kinesis_mock_server.py index af23e3940ef24..b9ce394e1415d 100644 --- a/localstack-core/localstack/services/kinesis/kinesis_mock_server.py +++ b/localstack-core/localstack/services/kinesis/kinesis_mock_server.py @@ -1,11 +1,16 @@ import logging import os import threading +from abc import abstractmethod from pathlib import Path from typing import Dict, List, Optional, Tuple from localstack import config -from localstack.services.kinesis.packages import kinesismock_package +from localstack.services.kinesis.packages import ( + KinesisMockEngine, + kinesismock_package, + kinesismock_scala_package, +) from localstack.utils.common import TMP_THREADS, ShellCommandThread, get_free_tcp_port, mkdir from localstack.utils.run import FuncThread from localstack.utils.serving import Server @@ -21,7 +26,7 @@ class KinesisMockServer(Server): def __init__( self, port: int, - js_path: Path, + exe_path: Path, latency: str, account_id: str, host: str = "localhost", @@ -32,7 +37,7 @@ def __init__( self._latency = latency self._data_dir = data_dir self._data_filename = f"{self._account_id}.json" - self._js_path = js_path + self._exe_path = exe_path self._log_level = log_level super().__init__(port, host) @@ -51,15 +56,9 @@ def do_start_thread(self) -> FuncThread: t.start() return t - def _create_shell_command(self) -> Tuple[List, Dict]: - """ - Helper method for creating kinesis mock invocation command - :return: returns a tuple containing the command list and a dictionary with the environment variables - """ - + @property + def _environment_variables(self) -> Dict: env_vars = { - # Use the `server.json` packaged next to the main.js - "KINESIS_MOCK_CERT_PATH": str((self._js_path.parent / "server.json").absolute()), "KINESIS_MOCK_PLAIN_PORT": self.port, # Each kinesis-mock instance listens to two ports - secure and insecure. # LocalStack uses only one - the insecure one. Block the secure port to avoid conflicts. @@ -91,13 +90,60 @@ def _create_shell_command(self) -> Tuple[List, Dict]: env_vars["PERSIST_INTERVAL"] = config.KINESIS_MOCK_PERSIST_INTERVAL env_vars["LOG_LEVEL"] = self._log_level - cmd = ["node", self._js_path] - return cmd, env_vars + + return env_vars + + @abstractmethod + def _create_shell_command(self) -> Tuple[List, Dict]: + """ + Helper method for creating kinesis mock invocation command + :return: returns a tuple containing the command list and a dictionary with the environment variables + """ + pass def _log_listener(self, line, **_kwargs): LOG.info(line.rstrip()) +class KinesisMockScalaServer(KinesisMockServer): + def _create_shell_command(self) -> Tuple[List, Dict]: + cmd = ["java", "-jar", *self._get_java_vm_options(), str(self._exe_path)] + return cmd, self._environment_variables + + @property + def _environment_variables(self) -> Dict: + default_env_vars = super()._environment_variables + kinesis_mock_installer = kinesismock_scala_package.get_installer() + return { + **default_env_vars, + **kinesis_mock_installer.get_java_env_vars(), + } + + def _get_java_vm_options(self) -> list[str]: + return [ + f"-Xms{config.KINESIS_MOCK_INITIAL_HEAP_SIZE}", + f"-Xmx{config.KINESIS_MOCK_MAXIMUM_HEAP_SIZE}", + "-XX:MaxGCPauseMillis=500", + "-XX:+ExitOnOutOfMemoryError", + ] + + +class KinesisMockNodeServer(KinesisMockServer): + @property + def _environment_variables(self) -> Dict: + node_env_vars = { + # Use the `server.json` packaged next to the main.js + "KINESIS_MOCK_CERT_PATH": str((self._exe_path.parent / "server.json").absolute()), + } + + default_env_vars = super()._environment_variables + return {**node_env_vars, **default_env_vars} + + def _create_shell_command(self) -> Tuple[List, Dict]: + cmd = ["node", self._exe_path] + return cmd, self._environment_variables + + class KinesisServerManager: default_startup_timeout = 60 @@ -136,8 +182,6 @@ def _create_kinesis_mock_server(self, account_id: str) -> KinesisMockServer: config.KINESIS_LATENCY -> configure stream latency (in milliseconds) """ port = get_free_tcp_port() - kinesismock_package.install() - kinesis_mock_js_path = Path(kinesismock_package.get_installer().get_executable_path()) # kinesis-mock stores state in json files .json, so we can dump everything into `kinesis/` persist_path = os.path.join(config.dirs.data, "kinesis") @@ -159,12 +203,31 @@ def _create_kinesis_mock_server(self, account_id: str) -> KinesisMockServer: log_level = "INFO" latency = config.KINESIS_LATENCY + "ms" - server = KinesisMockServer( + # Install the Scala Kinesis Mock build if specified in KINESIS_MOCK_PROVIDER_ENGINE + if KinesisMockEngine(config.KINESIS_MOCK_PROVIDER_ENGINE) == KinesisMockEngine.SCALA: + kinesismock_scala_package.install() + kinesis_mock_path = Path( + kinesismock_scala_package.get_installer().get_executable_path() + ) + + return KinesisMockScalaServer( + port=port, + exe_path=kinesis_mock_path, + log_level=log_level, + latency=latency, + data_dir=persist_path, + account_id=account_id, + ) + + # Otherwise, install the NodeJS version (default) + kinesismock_package.install() + kinesis_mock_path = Path(kinesismock_package.get_installer().get_executable_path()) + + return KinesisMockNodeServer( port=port, - js_path=kinesis_mock_js_path, + exe_path=kinesis_mock_path, log_level=log_level, latency=latency, data_dir=persist_path, account_id=account_id, ) - return server diff --git a/localstack-core/localstack/services/kinesis/packages.py b/localstack-core/localstack/services/kinesis/packages.py index d6b68dcd9d628..1d64bb4194b63 100644 --- a/localstack-core/localstack/services/kinesis/packages.py +++ b/localstack-core/localstack/services/kinesis/packages.py @@ -1,28 +1,82 @@ import os +from enum import StrEnum from functools import lru_cache -from typing import List +from typing import Any, List -from localstack.packages import Package -from localstack.packages.core import NodePackageInstaller +from localstack.packages import InstallTarget, Package +from localstack.packages.core import GitHubReleaseInstaller, NodePackageInstaller +from localstack.packages.java import JavaInstallerMixin, java_package -_KINESIS_MOCK_VERSION = os.environ.get("KINESIS_MOCK_VERSION") or "0.4.9" +_KINESIS_MOCK_VERSION = os.environ.get("KINESIS_MOCK_VERSION") or "0.4.12" -class KinesisMockPackage(Package[NodePackageInstaller]): - def __init__(self, default_version: str = _KINESIS_MOCK_VERSION): +class KinesisMockEngine(StrEnum): + NODE = "node" + SCALA = "scala" + + @classmethod + def _missing_(cls, value: str | Any) -> str: + # default to 'node' if invalid enum + if not isinstance(value, str): + return cls(cls.NODE) + return cls.__members__.get(value.upper(), cls.NODE) + + +class KinesisMockNodePackageInstaller(NodePackageInstaller): + def __init__(self, version: str): + super().__init__(package_name="kinesis-local", version=version) + + +class KinesisMockScalaPackageInstaller(JavaInstallerMixin, GitHubReleaseInstaller): + def __init__(self, version: str = _KINESIS_MOCK_VERSION): + super().__init__( + name="kinesis-local", tag=f"v{version}", github_slug="etspaceman/kinesis-mock" + ) + + # Kinesis Mock requires JRE 21+ + self.java_version = "21" + + def _get_github_asset_name(self) -> str: + return "kinesis-mock.jar" + + def _prepare_installation(self, target: InstallTarget) -> None: + java_package.get_installer(self.java_version).install(target) + + def get_java_home(self) -> str | None: + """Override to use the specific Java version""" + return java_package.get_installer(self.java_version).get_java_home() + + +class KinesisMockScalaPackage(Package[KinesisMockScalaPackageInstaller]): + def __init__( + self, + default_version: str = _KINESIS_MOCK_VERSION, + ): super().__init__(name="Kinesis Mock", default_version=default_version) @lru_cache - def _get_installer(self, version: str) -> NodePackageInstaller: - return KinesisMockPackageInstaller(version) + def _get_installer(self, version: str) -> KinesisMockScalaPackageInstaller: + return KinesisMockScalaPackageInstaller(version) def get_versions(self) -> List[str]: - return [_KINESIS_MOCK_VERSION] + return [_KINESIS_MOCK_VERSION] # Only supported on v0.4.12+ -class KinesisMockPackageInstaller(NodePackageInstaller): - def __init__(self, version: str): - super().__init__(package_name="kinesis-local", version=version) +class KinesisMockNodePackage(Package[KinesisMockNodePackageInstaller]): + def __init__( + self, + default_version: str = _KINESIS_MOCK_VERSION, + ): + super().__init__(name="Kinesis Mock", default_version=default_version) + + @lru_cache + def _get_installer(self, version: str) -> KinesisMockNodePackageInstaller: + return KinesisMockNodePackageInstaller(version) + + def get_versions(self) -> List[str]: + return [_KINESIS_MOCK_VERSION] -kinesismock_package = KinesisMockPackage() +# leave as 'kinesismock_package' for backwards compatability +kinesismock_package = KinesisMockNodePackage() +kinesismock_scala_package = KinesisMockScalaPackage() diff --git a/localstack-core/localstack/services/kinesis/plugins.py b/localstack-core/localstack/services/kinesis/plugins.py index 13f06b3e630ca..75249c9a2d904 100644 --- a/localstack-core/localstack/services/kinesis/plugins.py +++ b/localstack-core/localstack/services/kinesis/plugins.py @@ -1,8 +1,16 @@ +import localstack.config as config from localstack.packages import Package, package @package(name="kinesis-mock") def kinesismock_package() -> Package: - from localstack.services.kinesis.packages import kinesismock_package + from localstack.services.kinesis.packages import ( + KinesisMockEngine, + kinesismock_package, + kinesismock_scala_package, + ) + + if KinesisMockEngine(config.KINESIS_MOCK_PROVIDER_ENGINE) == KinesisMockEngine.SCALA: + return kinesismock_scala_package return kinesismock_package diff --git a/tests/aws/services/kinesis/test_kinesis.py b/tests/aws/services/kinesis/test_kinesis.py index 613ac9b7fc5e4..041b25bc28bcf 100644 --- a/tests/aws/services/kinesis/test_kinesis.py +++ b/tests/aws/services/kinesis/test_kinesis.py @@ -758,6 +758,25 @@ def test_subscribe_to_shard_with_java_sdk_v2_lambda( assert response_content == "ok" +@pytest.mark.skipif( + condition=is_aws_cloud(), + reason="Duplicate of all tests in TestKinesis. Since we cannot unmark test cases, only run against LocalStack.", +) +class TestKinesisMockScala(TestKinesis): + @pytest.fixture(autouse=True) + def set_kinesis_mock_scala_engine(self, monkeypatch): + monkeypatch.setattr(config, "KINESIS_MOCK_PROVIDER_ENGINE", "scala") + + @pytest.fixture(autouse=True, scope="function") + def override_snapshot_session(self, _snapshot_session): + # Replace the scope_key of the snapshot session to reference parent class' recorded snapshots + _snapshot_session.scope_key = _snapshot_session.scope_key.replace( + "TestKinesisMockScala", "TestKinesis" + ) + # Ensure we load in the previously recorded state now that the scope key has been updated + _snapshot_session.recorded_state = _snapshot_session._load_state() + + class TestKinesisPythonClient: @markers.skip_offline @markers.aws.only_localstack diff --git a/tests/unit/cli/test_lpm.py b/tests/unit/cli/test_lpm.py index 9463783c01059..605aac7ef00ad 100644 --- a/tests/unit/cli/test_lpm.py +++ b/tests/unit/cli/test_lpm.py @@ -102,3 +102,15 @@ def test_install_with_package(runner): result = runner.invoke(cli, ["install", "kinesis-mock"]) assert result.exit_code == 0 assert os.path.exists(kinesismock_package.get_installed_dir()) + + +@markers.skip_offline +def test_install_with_package_override(runner, monkeypatch): + from localstack import config + from localstack.services.kinesis.packages import kinesismock_scala_package + + monkeypatch.setattr(config, "KINESIS_MOCK_PROVIDER_ENGINE", "scala") + + result = runner.invoke(cli, ["install", "kinesis-mock"]) + assert result.exit_code == 0 + assert os.path.exists(kinesismock_scala_package.get_installed_dir())