localstack · thrau · Nov 26, 2021 · Nov 25, 2021 · Nov 25, 2021 · Nov 26, 2021
diff --git a/bin/docker-entrypoint.sh b/bin/docker-entrypoint.sh
@@ -5,6 +5,7 @@ shopt -s nullglob
 
 if [[ ! $INIT_SCRIPTS_PATH ]]
 then
+  # FIXME: move
   INIT_SCRIPTS_PATH=/docker-entrypoint-initaws.d
 fi
 if [[ ! $EDGE_PORT ]]

diff --git a/localstack/config.py b/localstack/config.py
@@ -7,7 +7,6 @@
 import subprocess
 import tempfile
 import time
-from os.path import expanduser
 from typing import Dict, List, Mapping
 
 import six
@@ -21,6 +20,7 @@
     DEFAULT_PORT_EDGE,
     DEFAULT_SERVICE_PORTS,
     FALSE_STRINGS,
+    INSTALL_DIR_INFRA,
     LOCALHOST,
     LOCALHOST_IP,
     LOG_LEVELS,
@@ -32,6 +32,118 @@
 load_start_time = time.time()
 
 
+class Directories:
+    """
+    Holds the different directories available to localstack. Some directories are shared between the host and the
+    localstack container, some live only on the host and some only in the container.
+
+    Attributes:
+        static_libs: container only; binaries and libraries statically packaged with the image
+        var_libs:    shared; binaries and libraries+data computed at runtime: lazy-loaded binaries, ssl cert, ...
+        cache:       shared; ephemeral data that has to persist across localstack runs and reboots
+        tmp:         shared; ephemeral data that has to persist across localstack runs but not reboots
+        functions:   shared; volume to communicate between host<->lambda containers
+        data:        shared; holds localstack state, pods, ...
+        config:      host only; pre-defined configuration values, cached credentials, machine id, ...
+        init:        shared; user-defined provisioning scripts executed in the container when it starts
+        logs:        shared; log files produced by localstack
+    """
+
+    static_libs: str
+    var_libs: str
+    cache: str
+    tmp: str
+    functions: str
+    data: str
+    config: str
+    init: str
+    logs: str
+
+    # these are the folders mounted into the container by default when the CLI is used
+    default_bind_mounts = ["var_libs", "cache", "tmp", "data", "init", "logs"]
+
+    def __init__(
+        self,
+        static_libs: str = None,
+        var_libs: str = None,
+        cache: str = None,
+        tmp: str = None,
+        functions: str = None,
+        data: str = None,
+        config: str = None,
+        init: str = None,
+        logs: str = None,
+    ) -> None:
+        super().__init__()
+        self.static_libs = static_libs
+        self.var_libs = var_libs
+        self.cache = cache
+        self.tmp = tmp
+        self.functions = functions
+        self.data = data
+        self.config = config
+        self.init = init
+        self.logs = logs
+
+    @staticmethod
+    def from_config():
+        """Returns Localstack directory paths from the config/environment variables defined by the config."""
+        return Directories(
+            static_libs=INSTALL_DIR_INFRA,
+            var_libs=TMP_FOLDER,  # TODO: add variable
+            cache=TMP_FOLDER,  # TODO: add variable
+            tmp=TMP_FOLDER,  # TODO: should inherit from root value for /var/lib/localstack (e.g., MOUNT_ROOT)
+            functions=HOST_TMP_FOLDER,  # TODO: rename variable/consider a volume
+            data=DATA_DIR,
+            config=None,  # TODO: will be introduced once .localstack config file has been refactored
+            init=None,  # TODO: introduce environment variable
+            logs=TMP_FOLDER,  # TODO: add variable
+        )
+
+    @staticmethod
+    def for_container() -> "Directories":
+        """
+        /var/lib/localstack. Returns Localstack directory paths as they are defined within the container. Everything
+        shared and writable lives in /var/lib/localstack or /tmp/localstack.
+
+        :returns: Directories object
+        """
+        return Directories(
+            static_libs=INSTALL_DIR_INFRA,
+            var_libs="/var/lib/localstack/var_libs",
+            cache="/var/lib/localstack/cache",
+            tmp=TMP_FOLDER,  # TODO: move to /var/lib/localstack/tmp
+            functions=HOST_TMP_FOLDER,  # TODO: move to /var/lib/localstack/tmp
+            data=DATA_DIR,  # TODO: move to /var/lib/localstack/data
+            config="/etc/localstack",  # TODO: will be introduced once .localstack config file has been refactored
+            logs="/var/lib/localstack/logs",
+            init="/docker-entrypoint-initaws.d",
+        )
+
+    def mkdirs(self):
+        for folder in [
+            self.static_libs,
+            self.var_libs,
+            self.cache,
+            self.tmp,
+            self.functions,
+            self.data,
+            self.config,
+            self.init,
+            self.logs,
+        ]:
+            if folder and not os.path.exists(folder):
+                try:
+                    os.makedirs(folder)
+                except Exception:
+                    # this can happen due to a race condition when starting
+                    # multiple processes in parallel. Should be safe to ignore
+                    pass
+
+    def __str__(self):
+        return str(self.__dict__)
+
+
 def eval_log_type(env_var_name):
     """get the log type from environment variable"""
     ls_log = os.environ.get(env_var_name, "").lower().strip()
@@ -133,16 +245,6 @@ def is_env_not_false(env_var_name):
 # folder for temporary files and data
 TMP_FOLDER = os.path.join(tempfile.gettempdir(), "localstack")
 
-# create folders
-for folder in [DATA_DIR, TMP_FOLDER]:
-    if folder and not os.path.exists(folder):
-        try:
-            os.makedirs(folder)
-        except Exception:
-            # this can happen due to a race condition when starting
-            # multiple processes in parallel. Should be safe to ignore
-            pass
-
 # fix for Mac OS, to be able to mount /var/folders in Docker
 if TMP_FOLDER.startswith("/var/folders/") and os.path.exists("/private%s" % TMP_FOLDER):
     TMP_FOLDER = "/private%s" % TMP_FOLDER
@@ -478,7 +580,7 @@ def in_docker():
 # local config file path in home directory
 CONFIG_FILE_PATH = os.path.join(TMP_FOLDER, ".localstack")
 if not is_in_docker:
-    CONFIG_FILE_PATH = os.path.join(expanduser("~"), ".localstack")
+    CONFIG_FILE_PATH = os.path.join(os.path.expanduser("~"), ".localstack")
 
 # set variables no_proxy, i.e., run internal service calls directly
 no_proxy = ",".join(set((LOCALSTACK_HOSTNAME, LOCALHOST, LOCALHOST_IP, "[::1]")))
@@ -673,3 +775,11 @@ def __iter__(self):
     LOG.debug(
         "Initializing the configuration took %s ms" % int((load_end_time - load_start_time) * 1000)
     )
+
+# initialize directories
+if is_in_docker:
+    dirs = Directories.for_container()
+else:
+    dirs = Directories.from_config()
+
+dirs.mkdirs()
diff --git a/localstack/constants.py b/localstack/constants.py
@@ -45,7 +45,9 @@
 MODULE_MAIN_PATH = os.path.dirname(os.path.realpath(__file__))
 # TODO rename to "ROOT_FOLDER"!
 LOCALSTACK_ROOT_FOLDER = os.path.realpath(os.path.join(MODULE_MAIN_PATH, ".."))
-INSTALL_DIR_INFRA = os.path.join(MODULE_MAIN_PATH, "infra")
+INSTALL_DIR_INFRA = os.path.join(
+    MODULE_MAIN_PATH, "infra"
+)  # FIXME: deprecated, use config.dirs.infra
 
 # virtualenv folder
 LOCALSTACK_VENV_FOLDER = os.environ.get("VIRTUAL_ENV")

diff --git a/localstack/contrib/thundra.py b/localstack/contrib/thundra.py
@@ -115,7 +115,7 @@ def _init_java_agent_configs():
         "https://repo.thundra.io/service/local/artifact/maven/redirect?"
         + "r=thundra-releases&g=io.thundra.agent&a=thundra-agent-lambda-bootstrap&v={v}"
     ).format(v=version)
-    THUNDRA_JAVA_AGENT_LOCAL_PATH = "%s/%s" % (config.TMP_FOLDER, jar_name)
+    THUNDRA_JAVA_AGENT_LOCAL_PATH = "%s/%s" % (config.dirs.tmp, jar_name)
 
 
 def _install_java_agent():
@@ -217,11 +217,11 @@ def _init_node_agent_configs() -> bool:
 
     THUNDRA_NODE_AGENT_VERSION = version.strip()
     THUNDRA_NODE_AGENT_LOCAL_PATH = "%s/thundra/node/%s/" % (
-        config.TMP_FOLDER,
+        config.dirs.tmp,
         THUNDRA_NODE_AGENT_VERSION,
     )
     THUNDRA_NODE_AGENT_LOCAL_PATH_ON_HOST = "%s/thundra/node/%s/" % (
-        config.HOST_TMP_FOLDER,
+        config.dirs.functions,
         THUNDRA_NODE_AGENT_VERSION,
     )
 
@@ -327,11 +327,11 @@ def _init_python_agent_configs() -> bool:
 
     THUNDRA_PYTHON_AGENT_VERSION = version.strip()
     THUNDRA_PYTHON_AGENT_LOCAL_PATH = "%s/thundra/python/%s/" % (
-        config.TMP_FOLDER,
+        config.dirs.tmp,
         THUNDRA_PYTHON_AGENT_VERSION,
     )
     THUNDRA_PYTHON_AGENT_LOCAL_PATH_ON_HOST = "%s/thundra/python/%s/" % (
-        config.HOST_TMP_FOLDER,
+        config.dirs.functions,
         THUNDRA_PYTHON_AGENT_VERSION,
     )
 

diff --git a/localstack/services/awslambda/lambda_api.py b/localstack/services/awslambda/lambda_api.py
@@ -77,8 +77,8 @@
 LAMBDA_POLICY_NAME_PATTERN = "lambda_policy_{name}_{qualifier}"
 # constants
 APP_NAME = "lambda_api"
-ARCHIVE_FILE_PATTERN = "%s/lambda.handler.*.jar" % config.TMP_FOLDER
-LAMBDA_SCRIPT_PATTERN = "%s/lambda_script_*.py" % config.TMP_FOLDER
+ARCHIVE_FILE_PATTERN = "%s/lambda.handler.*.jar" % config.dirs.tmp
+LAMBDA_SCRIPT_PATTERN = "%s/lambda_script_*.py" % config.dirs.tmp
 LAMBDA_ZIP_FILE_NAME = "original_lambda_archive.zip"
 LAMBDA_JAR_FILE_NAME = "original_lambda_archive.jar"
 
@@ -998,7 +998,7 @@ def set_archive_code(code, lambda_name, zip_file_content=None):
     latest_version = lambda_details.get_version(VERSION_LATEST)
     latest_version["CodeSize"] = len(zip_file_content)
     latest_version["CodeSha256"] = code_sha_256.decode("utf-8")
-    tmp_dir = "%s/zipfile.%s" % (config.TMP_FOLDER, short_uid())
+    tmp_dir = "%s/zipfile.%s" % (config.dirs.tmp, short_uid())
     mkdir(tmp_dir)
     tmp_file = "%s/%s" % (tmp_dir, LAMBDA_ZIP_FILE_NAME)
     save_file(tmp_file, zip_file_content)

diff --git a/localstack/services/awslambda/lambda_executors.py b/localstack/services/awslambda/lambda_executors.py
@@ -69,7 +69,7 @@
 LAMBDA_EXECUTOR_JAR = INSTALL_PATH_LOCALSTACK_FAT_JAR
 LAMBDA_EXECUTOR_CLASS = "cloud.localstack.LambdaExecutor"
 LAMBDA_HANDLER_ENV_VAR_NAME = "_HANDLER"
-EVENT_FILE_PATTERN = "%s/lambda.event.*.json" % config.TMP_FOLDER
+EVENT_FILE_PATTERN = "%s/lambda.event.*.json" % config.dirs.tmp
 
 LAMBDA_SERVER_UNIQUE_PORTS = 500
 LAMBDA_SERVER_PORT_OFFSET = 5000
@@ -1484,7 +1484,7 @@ def get_java_opts(cls):
 
     @classmethod
     def get_host_path_for_path_in_docker(cls, path):
-        return re.sub(r"^%s/(.*)$" % config.TMP_FOLDER, r"%s/\1" % config.HOST_TMP_FOLDER, path)
+        return re.sub(r"^%s/(.*)$" % config.dirs.tmp, r"%s/\1" % config.dirs.functions, path)
 
     @classmethod
     def format_windows_path(cls, path):
@@ -1540,7 +1540,7 @@ def get_java_classpath(cls, archive):
 
     @staticmethod
     def mountable_tmp_file():
-        f = os.path.join(config.TMP_FOLDER, short_uid())
+        f = os.path.join(config.dirs.tmp, short_uid())
         TMP_FILES.append(f)
         return f
 

diff --git a/localstack/services/cloudformation/deployment_utils.py b/localstack/services/cloudformation/deployment_utils.py
@@ -2,7 +2,7 @@
 import os
 from typing import Callable
 
-from localstack.constants import INSTALL_DIR_INFRA
+from localstack.config import dirs
 from localstack.utils import common
 
 # URL to "cfn-response" module which is required in some CF Lambdas
@@ -119,7 +119,7 @@ def _convert(params, **kwargs):
 
 
 def get_cfn_response_mod_file():
-    cfn_response_tmp_file = os.path.join(INSTALL_DIR_INFRA, "lambda.cfn-response.js")
+    cfn_response_tmp_file = os.path.join(dirs.static_libs, "lambda.cfn-response.js")
     if not os.path.exists(cfn_response_tmp_file):
         common.download(CFN_RESPONSE_MODULE_URL, cfn_response_tmp_file)
     return cfn_response_tmp_file

diff --git a/localstack/services/dynamodb/server.py b/localstack/services/dynamodb/server.py
@@ -3,8 +3,7 @@
 from typing import List, Optional
 
 from localstack import config
-from localstack.config import is_env_true
-from localstack.constants import MODULE_MAIN_PATH
+from localstack.config import dirs, is_env_true
 from localstack.services import install
 from localstack.utils.common import TMP_THREADS, ShellCommandThread, get_free_tcp_port, mkdir
 from localstack.utils.run import FuncThread
@@ -39,11 +38,11 @@ def in_memory(self):
 
     @property
     def jar_path(self) -> str:
-        return f"{MODULE_MAIN_PATH}/infra/dynamodb/DynamoDBLocal.jar"
+        return f"{dirs.static_libs}/dynamodb/DynamoDBLocal.jar"
 
     @property
     def library_path(self) -> str:
-        return f"{MODULE_MAIN_PATH}/infra/dynamodb/DynamoDBLocal_lib"
+        return f"{dirs.static_libs}/dynamodb/DynamoDBLocal_lib"
 
     def _create_shell_command(self) -> List[str]:
         cmd = [
@@ -96,8 +95,8 @@ def create_dynamodb_server(port=None) -> DynamodbServer:
 
     server = DynamodbServer(port)
 
-    if config.DATA_DIR:
-        ddb_data_dir = "%s/dynamodb" % config.DATA_DIR
+    if config.dirs.data:
+        ddb_data_dir = "%s/dynamodb" % config.dirs.data
         mkdir(ddb_data_dir)
         absolute_path = os.path.abspath(ddb_data_dir)
         server.db_path = absolute_path

diff --git a/localstack/services/es/cluster.py b/localstack/services/es/cluster.py
@@ -85,10 +85,10 @@ def resolve_directories(version: str, cluster_path: str, data_root: str = None)
     modules_dir = os.path.join(install_dir, "modules")
 
     if data_root is None:
-        if config.DATA_DIR:
-            data_root = config.DATA_DIR
+        if config.dirs.data:
+            data_root = config.dirs.data
         else:
-            data_root = config.TMP_FOLDER
+            data_root = config.dirs.tmp
 
     data_path = os.path.join(data_root, "elasticsearch", cluster_path)
 
@@ -106,7 +106,7 @@ def init_directories(dirs: Directories):
     LOG.debug("initializing elasticsearch directories %s", dirs)
     chmod_r(dirs.install, 0o777)
 
-    if not dirs.data.startswith(config.DATA_DIR):
+    if not dirs.data.startswith(config.dirs.data):
         # only clear previous data if it's not in DATA_DIR
         rm_rf(dirs.data)
 

diff --git a/localstack/services/events/events_listener.py b/localstack/services/events/events_listener.py
@@ -67,7 +67,7 @@ def _dump_events_to_files(events_with_added_uuid):
 
 
 def _get_events_tmp_dir():
-    return os.path.join(config.TMP_FOLDER, EVENTS_TMP_DIR)
+    return os.path.join(config.dirs.tmp, EVENTS_TMP_DIR)
 
 
 def get_scheduled_rule_func(data):

diff --git a/localstack/services/generic_proxy.py b/localstack/services/generic_proxy.py
@@ -641,7 +641,7 @@ async def _accept_connection2(self, protocol_factory, conn, extra, sslcontext, *
 
 
 def get_cert_pem_file_path():
-    return os.path.join(config.TMP_FOLDER, SERVER_CERT_PEM_FILE)
+    return os.path.join(config.dirs.tmp, SERVER_CERT_PEM_FILE)
 
 
 def start_proxy_server(

diff --git a/localstack/services/infra.py b/localstack/services/infra.py
@@ -378,11 +378,7 @@ def start_infra(asynchronous=False, apis=None):
                 % config.LAMBDA_EXECUTOR
             )
 
-        if (
-            is_in_docker
-            and not config.LAMBDA_REMOTE_DOCKER
-            and not os.environ.get("HOST_TMP_FOLDER")
-        ):
+        if is_in_docker and not config.LAMBDA_REMOTE_DOCKER and not config.dirs.functions:
             print(
                 "!WARNING! - Looks like you have configured $LAMBDA_REMOTE_DOCKER=0 - "
                 "please make sure to configure $HOST_TMP_FOLDER to point to your host's $TMPDIR"
@@ -513,7 +509,7 @@ def start_runtime_components():
     thread = start_runtime_components()
     preload_services()
 
-    if config.DATA_DIR:
+    if config.dirs.data:
         persistence.save_startup_info()
 
     print(READY_MARKER_OUTPUT)