diff --git a/bin/docker-entrypoint.sh b/bin/docker-entrypoint.sh index 12df103dad0b0..0ee667ae80ad2 100755 --- a/bin/docker-entrypoint.sh +++ b/bin/docker-entrypoint.sh @@ -5,6 +5,7 @@ shopt -s nullglob if [[ ! $INIT_SCRIPTS_PATH ]] then + # FIXME: move INIT_SCRIPTS_PATH=/docker-entrypoint-initaws.d fi if [[ ! $EDGE_PORT ]] diff --git a/localstack/config.py b/localstack/config.py index 3be8633fdaeee..8cb26e50f769d 100644 --- a/localstack/config.py +++ b/localstack/config.py @@ -7,7 +7,6 @@ import subprocess import tempfile import time -from os.path import expanduser from typing import Dict, List, Mapping import six @@ -21,6 +20,7 @@ DEFAULT_PORT_EDGE, DEFAULT_SERVICE_PORTS, FALSE_STRINGS, + INSTALL_DIR_INFRA, LOCALHOST, LOCALHOST_IP, LOG_LEVELS, @@ -32,6 +32,118 @@ load_start_time = time.time() +class Directories: + """ + Holds the different directories available to localstack. Some directories are shared between the host and the + localstack container, some live only on the host and some only in the container. + + Attributes: + static_libs: container only; binaries and libraries statically packaged with the image + var_libs: shared; binaries and libraries+data computed at runtime: lazy-loaded binaries, ssl cert, ... + cache: shared; ephemeral data that has to persist across localstack runs and reboots + tmp: shared; ephemeral data that has to persist across localstack runs but not reboots + functions: shared; volume to communicate between host<->lambda containers + data: shared; holds localstack state, pods, ... + config: host only; pre-defined configuration values, cached credentials, machine id, ... + init: shared; user-defined provisioning scripts executed in the container when it starts + logs: shared; log files produced by localstack + """ + + static_libs: str + var_libs: str + cache: str + tmp: str + functions: str + data: str + config: str + init: str + logs: str + + # these are the folders mounted into the container by default when the CLI is used + default_bind_mounts = ["var_libs", "cache", "tmp", "data", "init", "logs"] + + def __init__( + self, + static_libs: str = None, + var_libs: str = None, + cache: str = None, + tmp: str = None, + functions: str = None, + data: str = None, + config: str = None, + init: str = None, + logs: str = None, + ) -> None: + super().__init__() + self.static_libs = static_libs + self.var_libs = var_libs + self.cache = cache + self.tmp = tmp + self.functions = functions + self.data = data + self.config = config + self.init = init + self.logs = logs + + @staticmethod + def from_config(): + """Returns Localstack directory paths from the config/environment variables defined by the config.""" + return Directories( + static_libs=INSTALL_DIR_INFRA, + var_libs=TMP_FOLDER, # TODO: add variable + cache=TMP_FOLDER, # TODO: add variable + tmp=TMP_FOLDER, # TODO: should inherit from root value for /var/lib/localstack (e.g., MOUNT_ROOT) + functions=HOST_TMP_FOLDER, # TODO: rename variable/consider a volume + data=DATA_DIR, + config=None, # TODO: will be introduced once .localstack config file has been refactored + init=None, # TODO: introduce environment variable + logs=TMP_FOLDER, # TODO: add variable + ) + + @staticmethod + def for_container() -> "Directories": + """ + /var/lib/localstack. Returns Localstack directory paths as they are defined within the container. Everything + shared and writable lives in /var/lib/localstack or /tmp/localstack. + + :returns: Directories object + """ + return Directories( + static_libs=INSTALL_DIR_INFRA, + var_libs="/var/lib/localstack/var_libs", + cache="/var/lib/localstack/cache", + tmp=TMP_FOLDER, # TODO: move to /var/lib/localstack/tmp + functions=HOST_TMP_FOLDER, # TODO: move to /var/lib/localstack/tmp + data=DATA_DIR, # TODO: move to /var/lib/localstack/data + config="/etc/localstack", # TODO: will be introduced once .localstack config file has been refactored + logs="/var/lib/localstack/logs", + init="/docker-entrypoint-initaws.d", + ) + + def mkdirs(self): + for folder in [ + self.static_libs, + self.var_libs, + self.cache, + self.tmp, + self.functions, + self.data, + self.config, + self.init, + self.logs, + ]: + if folder and not os.path.exists(folder): + try: + os.makedirs(folder) + except Exception: + # this can happen due to a race condition when starting + # multiple processes in parallel. Should be safe to ignore + pass + + def __str__(self): + return str(self.__dict__) + + def eval_log_type(env_var_name): """get the log type from environment variable""" ls_log = os.environ.get(env_var_name, "").lower().strip() @@ -133,16 +245,6 @@ def is_env_not_false(env_var_name): # folder for temporary files and data TMP_FOLDER = os.path.join(tempfile.gettempdir(), "localstack") -# create folders -for folder in [DATA_DIR, TMP_FOLDER]: - if folder and not os.path.exists(folder): - try: - os.makedirs(folder) - except Exception: - # this can happen due to a race condition when starting - # multiple processes in parallel. Should be safe to ignore - pass - # fix for Mac OS, to be able to mount /var/folders in Docker if TMP_FOLDER.startswith("/var/folders/") and os.path.exists("/private%s" % TMP_FOLDER): TMP_FOLDER = "/private%s" % TMP_FOLDER @@ -478,7 +580,7 @@ def in_docker(): # local config file path in home directory CONFIG_FILE_PATH = os.path.join(TMP_FOLDER, ".localstack") if not is_in_docker: - CONFIG_FILE_PATH = os.path.join(expanduser("~"), ".localstack") + CONFIG_FILE_PATH = os.path.join(os.path.expanduser("~"), ".localstack") # set variables no_proxy, i.e., run internal service calls directly no_proxy = ",".join(set((LOCALSTACK_HOSTNAME, LOCALHOST, LOCALHOST_IP, "[::1]"))) @@ -673,3 +775,11 @@ def __iter__(self): LOG.debug( "Initializing the configuration took %s ms" % int((load_end_time - load_start_time) * 1000) ) + +# initialize directories +if is_in_docker: + dirs = Directories.for_container() +else: + dirs = Directories.from_config() + +dirs.mkdirs() diff --git a/localstack/constants.py b/localstack/constants.py index 0a2dfdb6d3f8c..f6c2bbddc34ae 100644 --- a/localstack/constants.py +++ b/localstack/constants.py @@ -45,7 +45,9 @@ MODULE_MAIN_PATH = os.path.dirname(os.path.realpath(__file__)) # TODO rename to "ROOT_FOLDER"! LOCALSTACK_ROOT_FOLDER = os.path.realpath(os.path.join(MODULE_MAIN_PATH, "..")) -INSTALL_DIR_INFRA = os.path.join(MODULE_MAIN_PATH, "infra") +INSTALL_DIR_INFRA = os.path.join( + MODULE_MAIN_PATH, "infra" +) # FIXME: deprecated, use config.dirs.infra # virtualenv folder LOCALSTACK_VENV_FOLDER = os.environ.get("VIRTUAL_ENV") diff --git a/localstack/contrib/thundra.py b/localstack/contrib/thundra.py index 2c4591a5bc312..45dde1b63010d 100644 --- a/localstack/contrib/thundra.py +++ b/localstack/contrib/thundra.py @@ -115,7 +115,7 @@ def _init_java_agent_configs(): "https://repo.thundra.io/service/local/artifact/maven/redirect?" + "r=thundra-releases&g=io.thundra.agent&a=thundra-agent-lambda-bootstrap&v={v}" ).format(v=version) - THUNDRA_JAVA_AGENT_LOCAL_PATH = "%s/%s" % (config.TMP_FOLDER, jar_name) + THUNDRA_JAVA_AGENT_LOCAL_PATH = "%s/%s" % (config.dirs.tmp, jar_name) def _install_java_agent(): @@ -217,11 +217,11 @@ def _init_node_agent_configs() -> bool: THUNDRA_NODE_AGENT_VERSION = version.strip() THUNDRA_NODE_AGENT_LOCAL_PATH = "%s/thundra/node/%s/" % ( - config.TMP_FOLDER, + config.dirs.tmp, THUNDRA_NODE_AGENT_VERSION, ) THUNDRA_NODE_AGENT_LOCAL_PATH_ON_HOST = "%s/thundra/node/%s/" % ( - config.HOST_TMP_FOLDER, + config.dirs.functions, THUNDRA_NODE_AGENT_VERSION, ) @@ -327,11 +327,11 @@ def _init_python_agent_configs() -> bool: THUNDRA_PYTHON_AGENT_VERSION = version.strip() THUNDRA_PYTHON_AGENT_LOCAL_PATH = "%s/thundra/python/%s/" % ( - config.TMP_FOLDER, + config.dirs.tmp, THUNDRA_PYTHON_AGENT_VERSION, ) THUNDRA_PYTHON_AGENT_LOCAL_PATH_ON_HOST = "%s/thundra/python/%s/" % ( - config.HOST_TMP_FOLDER, + config.dirs.functions, THUNDRA_PYTHON_AGENT_VERSION, ) diff --git a/localstack/services/awslambda/lambda_api.py b/localstack/services/awslambda/lambda_api.py index 98db8147b8265..9b68b93294be3 100644 --- a/localstack/services/awslambda/lambda_api.py +++ b/localstack/services/awslambda/lambda_api.py @@ -77,8 +77,8 @@ LAMBDA_POLICY_NAME_PATTERN = "lambda_policy_{name}_{qualifier}" # constants APP_NAME = "lambda_api" -ARCHIVE_FILE_PATTERN = "%s/lambda.handler.*.jar" % config.TMP_FOLDER -LAMBDA_SCRIPT_PATTERN = "%s/lambda_script_*.py" % config.TMP_FOLDER +ARCHIVE_FILE_PATTERN = "%s/lambda.handler.*.jar" % config.dirs.tmp +LAMBDA_SCRIPT_PATTERN = "%s/lambda_script_*.py" % config.dirs.tmp LAMBDA_ZIP_FILE_NAME = "original_lambda_archive.zip" LAMBDA_JAR_FILE_NAME = "original_lambda_archive.jar" @@ -998,7 +998,7 @@ def set_archive_code(code, lambda_name, zip_file_content=None): latest_version = lambda_details.get_version(VERSION_LATEST) latest_version["CodeSize"] = len(zip_file_content) latest_version["CodeSha256"] = code_sha_256.decode("utf-8") - tmp_dir = "%s/zipfile.%s" % (config.TMP_FOLDER, short_uid()) + tmp_dir = "%s/zipfile.%s" % (config.dirs.tmp, short_uid()) mkdir(tmp_dir) tmp_file = "%s/%s" % (tmp_dir, LAMBDA_ZIP_FILE_NAME) save_file(tmp_file, zip_file_content) diff --git a/localstack/services/awslambda/lambda_executors.py b/localstack/services/awslambda/lambda_executors.py index 90c4e7a9d35bc..200080acf3902 100644 --- a/localstack/services/awslambda/lambda_executors.py +++ b/localstack/services/awslambda/lambda_executors.py @@ -69,7 +69,7 @@ LAMBDA_EXECUTOR_JAR = INSTALL_PATH_LOCALSTACK_FAT_JAR LAMBDA_EXECUTOR_CLASS = "cloud.localstack.LambdaExecutor" LAMBDA_HANDLER_ENV_VAR_NAME = "_HANDLER" -EVENT_FILE_PATTERN = "%s/lambda.event.*.json" % config.TMP_FOLDER +EVENT_FILE_PATTERN = "%s/lambda.event.*.json" % config.dirs.tmp LAMBDA_SERVER_UNIQUE_PORTS = 500 LAMBDA_SERVER_PORT_OFFSET = 5000 @@ -1484,7 +1484,7 @@ def get_java_opts(cls): @classmethod def get_host_path_for_path_in_docker(cls, path): - return re.sub(r"^%s/(.*)$" % config.TMP_FOLDER, r"%s/\1" % config.HOST_TMP_FOLDER, path) + return re.sub(r"^%s/(.*)$" % config.dirs.tmp, r"%s/\1" % config.dirs.functions, path) @classmethod def format_windows_path(cls, path): @@ -1540,7 +1540,7 @@ def get_java_classpath(cls, archive): @staticmethod def mountable_tmp_file(): - f = os.path.join(config.TMP_FOLDER, short_uid()) + f = os.path.join(config.dirs.tmp, short_uid()) TMP_FILES.append(f) return f diff --git a/localstack/services/cloudformation/deployment_utils.py b/localstack/services/cloudformation/deployment_utils.py index 8e2b89ecefa5e..69000527b39fc 100644 --- a/localstack/services/cloudformation/deployment_utils.py +++ b/localstack/services/cloudformation/deployment_utils.py @@ -2,7 +2,7 @@ import os from typing import Callable -from localstack.constants import INSTALL_DIR_INFRA +from localstack.config import dirs from localstack.utils import common # URL to "cfn-response" module which is required in some CF Lambdas @@ -119,7 +119,7 @@ def _convert(params, **kwargs): def get_cfn_response_mod_file(): - cfn_response_tmp_file = os.path.join(INSTALL_DIR_INFRA, "lambda.cfn-response.js") + cfn_response_tmp_file = os.path.join(dirs.static_libs, "lambda.cfn-response.js") if not os.path.exists(cfn_response_tmp_file): common.download(CFN_RESPONSE_MODULE_URL, cfn_response_tmp_file) return cfn_response_tmp_file diff --git a/localstack/services/dynamodb/server.py b/localstack/services/dynamodb/server.py index efe38f5e54bca..4063d89095824 100644 --- a/localstack/services/dynamodb/server.py +++ b/localstack/services/dynamodb/server.py @@ -3,8 +3,7 @@ from typing import List, Optional from localstack import config -from localstack.config import is_env_true -from localstack.constants import MODULE_MAIN_PATH +from localstack.config import dirs, is_env_true from localstack.services import install from localstack.utils.common import TMP_THREADS, ShellCommandThread, get_free_tcp_port, mkdir from localstack.utils.run import FuncThread @@ -39,11 +38,11 @@ def in_memory(self): @property def jar_path(self) -> str: - return f"{MODULE_MAIN_PATH}/infra/dynamodb/DynamoDBLocal.jar" + return f"{dirs.static_libs}/dynamodb/DynamoDBLocal.jar" @property def library_path(self) -> str: - return f"{MODULE_MAIN_PATH}/infra/dynamodb/DynamoDBLocal_lib" + return f"{dirs.static_libs}/dynamodb/DynamoDBLocal_lib" def _create_shell_command(self) -> List[str]: cmd = [ @@ -96,8 +95,8 @@ def create_dynamodb_server(port=None) -> DynamodbServer: server = DynamodbServer(port) - if config.DATA_DIR: - ddb_data_dir = "%s/dynamodb" % config.DATA_DIR + if config.dirs.data: + ddb_data_dir = "%s/dynamodb" % config.dirs.data mkdir(ddb_data_dir) absolute_path = os.path.abspath(ddb_data_dir) server.db_path = absolute_path diff --git a/localstack/services/es/cluster.py b/localstack/services/es/cluster.py index 122fed2e235ed..853b06162b9f4 100644 --- a/localstack/services/es/cluster.py +++ b/localstack/services/es/cluster.py @@ -85,10 +85,10 @@ def resolve_directories(version: str, cluster_path: str, data_root: str = None) modules_dir = os.path.join(install_dir, "modules") if data_root is None: - if config.DATA_DIR: - data_root = config.DATA_DIR + if config.dirs.data: + data_root = config.dirs.data else: - data_root = config.TMP_FOLDER + data_root = config.dirs.tmp data_path = os.path.join(data_root, "elasticsearch", cluster_path) @@ -106,7 +106,7 @@ def init_directories(dirs: Directories): LOG.debug("initializing elasticsearch directories %s", dirs) chmod_r(dirs.install, 0o777) - if not dirs.data.startswith(config.DATA_DIR): + if not dirs.data.startswith(config.dirs.data): # only clear previous data if it's not in DATA_DIR rm_rf(dirs.data) diff --git a/localstack/services/events/events_listener.py b/localstack/services/events/events_listener.py index b23a32ed74096..8cb27dfcc41cb 100644 --- a/localstack/services/events/events_listener.py +++ b/localstack/services/events/events_listener.py @@ -67,7 +67,7 @@ def _dump_events_to_files(events_with_added_uuid): def _get_events_tmp_dir(): - return os.path.join(config.TMP_FOLDER, EVENTS_TMP_DIR) + return os.path.join(config.dirs.tmp, EVENTS_TMP_DIR) def get_scheduled_rule_func(data): diff --git a/localstack/services/generic_proxy.py b/localstack/services/generic_proxy.py index 3f5a4fb2fcc7f..ec137b2057591 100644 --- a/localstack/services/generic_proxy.py +++ b/localstack/services/generic_proxy.py @@ -641,7 +641,7 @@ async def _accept_connection2(self, protocol_factory, conn, extra, sslcontext, * def get_cert_pem_file_path(): - return os.path.join(config.TMP_FOLDER, SERVER_CERT_PEM_FILE) + return os.path.join(config.dirs.tmp, SERVER_CERT_PEM_FILE) def start_proxy_server( diff --git a/localstack/services/infra.py b/localstack/services/infra.py index 6541413f5a829..666591e943a84 100644 --- a/localstack/services/infra.py +++ b/localstack/services/infra.py @@ -378,11 +378,7 @@ def start_infra(asynchronous=False, apis=None): % config.LAMBDA_EXECUTOR ) - if ( - is_in_docker - and not config.LAMBDA_REMOTE_DOCKER - and not os.environ.get("HOST_TMP_FOLDER") - ): + if is_in_docker and not config.LAMBDA_REMOTE_DOCKER and not config.dirs.functions: print( "!WARNING! - Looks like you have configured $LAMBDA_REMOTE_DOCKER=0 - " "please make sure to configure $HOST_TMP_FOLDER to point to your host's $TMPDIR" @@ -513,7 +509,7 @@ def start_runtime_components(): thread = start_runtime_components() preload_services() - if config.DATA_DIR: + if config.dirs.data: persistence.save_startup_info() print(READY_MARKER_OUTPUT) diff --git a/localstack/services/install.py b/localstack/services/install.py index 2cd6520cb3306..0bc500a89a3e8 100644 --- a/localstack/services/install.py +++ b/localstack/services/install.py @@ -17,7 +17,7 @@ from plugin import Plugin, PluginManager from localstack import config -from localstack.config import is_env_true +from localstack.config import dirs, is_env_true from localstack.constants import ( DEFAULT_SERVICE_PORTS, DYNAMODB_JAR_URL, @@ -25,7 +25,6 @@ ELASTICSEARCH_DEFAULT_VERSION, ELASTICSEARCH_DELETE_MODULES, ELASTICSEARCH_PLUGIN_LIST, - INSTALL_DIR_INFRA, KMS_URL_PATTERN, LOCALSTACK_MAVEN_VERSION, MODULE_MAIN_PATH, @@ -54,26 +53,26 @@ LOG = logging.getLogger(__name__) -INSTALL_DIR_NPM = "%s/node_modules" % MODULE_MAIN_PATH -INSTALL_DIR_DDB = "%s/dynamodb" % INSTALL_DIR_INFRA -INSTALL_DIR_KCL = "%s/amazon-kinesis-client" % INSTALL_DIR_INFRA -INSTALL_DIR_STEPFUNCTIONS = "%s/stepfunctions" % INSTALL_DIR_INFRA -INSTALL_DIR_KMS = "%s/kms" % INSTALL_DIR_INFRA -INSTALL_DIR_ELASTICMQ = "%s/elasticmq" % INSTALL_DIR_INFRA -INSTALL_PATH_LOCALSTACK_FAT_JAR = "%s/localstack-utils-fat.jar" % INSTALL_DIR_INFRA +INSTALL_DIR_NPM = "%s/node_modules" % MODULE_MAIN_PATH # FIXME: migrate to infra +INSTALL_DIR_DDB = "%s/dynamodb" % dirs.static_libs +INSTALL_DIR_KCL = "%s/amazon-kinesis-client" % dirs.static_libs +INSTALL_DIR_STEPFUNCTIONS = "%s/stepfunctions" % dirs.static_libs +INSTALL_DIR_KMS = "%s/kms" % dirs.static_libs +INSTALL_DIR_ELASTICMQ = "%s/elasticmq" % dirs.static_libs +INSTALL_PATH_LOCALSTACK_FAT_JAR = "%s/localstack-utils-fat.jar" % dirs.static_libs INSTALL_PATH_DDB_JAR = os.path.join(INSTALL_DIR_DDB, "DynamoDBLocal.jar") INSTALL_PATH_KCL_JAR = os.path.join(INSTALL_DIR_KCL, "aws-java-sdk-sts.jar") INSTALL_PATH_STEPFUNCTIONS_JAR = os.path.join(INSTALL_DIR_STEPFUNCTIONS, "StepFunctionsLocal.jar") INSTALL_PATH_KMS_BINARY_PATTERN = os.path.join(INSTALL_DIR_KMS, "local-kms..bin") INSTALL_PATH_ELASTICMQ_JAR = os.path.join(INSTALL_DIR_ELASTICMQ, "elasticmq-server.jar") INSTALL_PATH_KINESALITE_CLI = os.path.join(INSTALL_DIR_NPM, "kinesalite", "cli.js") -INSTALL_PATH_KINESIS_MOCK = os.path.join(INSTALL_DIR_INFRA, "kinesis-mock") +INSTALL_PATH_KINESIS_MOCK = os.path.join(dirs.static_libs, "kinesis-mock") URL_LOCALSTACK_FAT_JAR = ( "https://repo1.maven.org/maven2/" + "cloud/localstack/localstack-utils/{v}/localstack-utils-{v}-fat.jar" ).format(v=LOCALSTACK_MAVEN_VERSION) -MARKER_FILE_LIGHT_VERSION = "%s/.light-version" % INSTALL_DIR_INFRA +MARKER_FILE_LIGHT_VERSION = "%s/.light-version" % dirs.static_libs IMAGE_NAME_SFN_LOCAL = "amazon/aws-stepfunctions-local" ARTIFACTS_REPO = "https://github.com/localstack/localstack-artifacts" SFN_PATCH_CLASS1 = "com/amazonaws/stepfunctions/local/runtime/Config.class" @@ -108,7 +107,7 @@ # GO Lambda runtime GO_RUNTIME_VERSION = "0.4.0" GO_RUNTIME_DOWNLOAD_URL_TEMPLATE = "https://github.com/localstack/awslamba-go-runtime/releases/download/v{version}/awslamba-go-runtime-{version}-{os}-{arch}.tar.gz" -GO_INSTALL_FOLDER = os.path.join(config.TMP_FOLDER, "awslamba-go-runtime") +GO_INSTALL_FOLDER = os.path.join(config.dirs.tmp, "awslamba-go-runtime") GO_LAMBDA_RUNTIME = os.path.join(GO_INSTALL_FOLDER, "aws-lambda-mock") GO_LAMBDA_MOCKSERVER = os.path.join(GO_INSTALL_FOLDER, "mockserver") @@ -117,7 +116,7 @@ TERRAFORM_URL_TEMPLATE = ( "https://releases.hashicorp.com/terraform/{version}/terraform_{version}_{os}_{arch}.zip" ) -TERRAFORM_BIN = os.path.join(INSTALL_DIR_INFRA, f"terraform-{TERRAFORM_VERSION}", "terraform") +TERRAFORM_BIN = os.path.join(dirs.static_libs, f"terraform-{TERRAFORM_VERSION}", "terraform") def get_elasticsearch_install_version(version: str) -> str: @@ -134,10 +133,10 @@ def get_elasticsearch_install_dir(version: str) -> str: if version == ELASTICSEARCH_DEFAULT_VERSION and not os.path.exists(MARKER_FILE_LIGHT_VERSION): # install the default version into a subfolder of the code base - install_dir = os.path.join(INSTALL_DIR_INFRA, "elasticsearch") + install_dir = os.path.join(dirs.static_libs, "elasticsearch") else: # put all other versions into the TMP_FOLDER - install_dir = os.path.join(config.TMP_FOLDER, "elasticsearch", version) + install_dir = os.path.join(config.dirs.tmp, "elasticsearch", version) return install_dir @@ -157,7 +156,7 @@ def install_elasticsearch(version=None): install_dir_parent = os.path.dirname(install_dir) mkdir(install_dir_parent) # download and extract archive - tmp_archive = os.path.join(config.TMP_FOLDER, "localstack.%s" % os.path.basename(es_url)) + tmp_archive = os.path.join(config.dirs.tmp, "localstack.%s" % os.path.basename(es_url)) download_and_extract_with_retry(es_url, tmp_archive, install_dir_parent) elasticsearch_dir = glob.glob(os.path.join(install_dir_parent, "elasticsearch*")) if not elasticsearch_dir: @@ -223,7 +222,7 @@ def install_elasticmq(): log_install_msg("ElasticMQ") mkdir(INSTALL_DIR_ELASTICMQ) # download archive - tmp_archive = os.path.join(config.TMP_FOLDER, "elasticmq-server.jar") + tmp_archive = os.path.join(config.dirs.tmp, "elasticmq-server.jar") if not os.path.exists(tmp_archive): download(ELASTICMQ_JAR_URL, tmp_archive) shutil.copy(tmp_archive, INSTALL_DIR_ELASTICMQ) @@ -331,13 +330,13 @@ def install_stepfunctions_local(): ) time.sleep(5) DOCKER_CLIENT.copy_from_container( - docker_name, local_path=INSTALL_DIR_INFRA, container_path="/home/stepfunctionslocal/" + docker_name, local_path=dirs.static_libs, container_path="/home/stepfunctionslocal/" ) - path = Path(f"{INSTALL_DIR_INFRA}/stepfunctionslocal/") + path = Path(f"{dirs.static_libs}/stepfunctionslocal/") for file in path.glob("*.jar"): file.rename(Path(INSTALL_DIR_STEPFUNCTIONS) / file.name) - rm_rf("%s/stepfunctionslocal" % INSTALL_DIR_INFRA) + rm_rf("%s/stepfunctionslocal" % dirs.static_libs) # apply patches for patch_class, patch_url in ( (SFN_PATCH_CLASS1, SFN_PATCH_CLASS_URL1), diff --git a/localstack/services/kinesis/kinesis_starter.py b/localstack/services/kinesis/kinesis_starter.py index 0894b966829a1..73b89be8b0c5f 100644 --- a/localstack/services/kinesis/kinesis_starter.py +++ b/localstack/services/kinesis/kinesis_starter.py @@ -30,11 +30,11 @@ def apply_patches_kinesalite(): files = [ - "%s/node_modules/kinesalite/validations/decreaseStreamRetentionPeriod.js", - "%s/node_modules/kinesalite/validations/increaseStreamRetentionPeriod.js", + "%s/kinesalite/validations/decreaseStreamRetentionPeriod.js", + "%s/kinesalite/validations/increaseStreamRetentionPeriod.js", ] for file_path in files: - file_path = file_path % MODULE_MAIN_PATH + file_path = file_path % install.INSTALL_DIR_NPM replace_in_file("lessThanOrEqual: 168", "lessThanOrEqual: 8760", file_path) @@ -85,8 +85,8 @@ def start_kinesis_mock(port=None, asynchronous=False, update_listener=None): global PORT_KINESIS_BACKEND PORT_KINESIS_BACKEND = backend_port kinesis_data_dir_param = "" - if config.DATA_DIR: - kinesis_data_dir = "%s/kinesis" % config.DATA_DIR + if config.dirs.data: + kinesis_data_dir = "%s/kinesis" % config.dirs.data mkdir(kinesis_data_dir) kinesis_data_dir_param = "SHOULD_PERSIST_DATA=true PERSIST_PATH=%s" % kinesis_data_dir if not config.LS_LOG: @@ -150,8 +150,8 @@ def start_kinesalite(port=None, asynchronous=False, update_listener=None): PORT_KINESIS_BACKEND = backend_port latency = config.KINESIS_LATENCY kinesis_data_dir_param = "" - if config.DATA_DIR: - kinesis_data_dir = "%s/kinesis" % config.DATA_DIR + if config.dirs.data: + kinesis_data_dir = "%s/kinesis" % config.dirs.data mkdir(kinesis_data_dir) kinesis_data_dir_param = "--path %s" % kinesis_data_dir cmd = ( diff --git a/localstack/services/kms/kms_starter.py b/localstack/services/kms/kms_starter.py index d0b7a695ac3e9..8fcc49eb40c76 100644 --- a/localstack/services/kms/kms_starter.py +++ b/localstack/services/kms/kms_starter.py @@ -31,8 +31,8 @@ def start_kms_local(port=None, backend_port=None, asynchronous=None, update_list "KMS_ACCOUNT_ID": TEST_AWS_ACCOUNT_ID, "ACCOUNT_ID": TEST_AWS_ACCOUNT_ID, } - if config.DATA_DIR: - env_vars["KMS_DATA_PATH"] = config.DATA_DIR + if config.dirs.data: + env_vars["KMS_DATA_PATH"] = config.dirs.data result = do_run(kms_binary, asynchronous, env_vars=env_vars) wait_for_port_open(backend_port) return result diff --git a/localstack/services/ses/ses_starter.py b/localstack/services/ses/ses_starter.py index b37b47f83511d..ec137f29ace9a 100644 --- a/localstack/services/ses/ses_starter.py +++ b/localstack/services/ses/ses_starter.py @@ -149,7 +149,7 @@ def get_email_log_object(source, region, destinations, subject=None, body=None, return email def log_email_to_data_dir(id, email): - ses_dir = os.path.join(config.DATA_DIR or config.TMP_FOLDER, "ses") + ses_dir = os.path.join(config.dirs.data or config.dirs.tmp, "ses") mkdir(ses_dir) with open(os.path.join(ses_dir, id + ".json"), "w") as f: @@ -188,7 +188,7 @@ def send_templated_email_save_contents( self, source, template, template_data, destinations, region ) - ses_dir = os.path.join(config.DATA_DIR or config.TMP_FOLDER, "ses") + ses_dir = os.path.join(config.dirs.data or config.dirs.tmp, "ses") mkdir(ses_dir) with open(os.path.join(ses_dir, message.id + ".json"), "w") as f: diff --git a/localstack/services/sqs/elasticmq.py b/localstack/services/sqs/elasticmq.py index 36c882a1823a5..2e3f20993066c 100644 --- a/localstack/services/sqs/elasticmq.py +++ b/localstack/services/sqs/elasticmq.py @@ -52,7 +52,7 @@ def do_start_thread(self) -> FuncThread: ) # create temporary config - config_file = os.path.join(config.TMP_FOLDER, "sqs.%s.conf" % short_uid()) + config_file = os.path.join(config.dirs.tmp, "sqs.%s.conf" % short_uid()) LOG.debug("saving config file to %s:\n%s", config_file, config_params) TMP_FILES.append(config_file) save_file(config_file, config_params) diff --git a/localstack/utils/analytics/event_publisher.py b/localstack/utils/analytics/event_publisher.py index 8d968d5345cff..a71c0a95e3262 100644 --- a/localstack/utils/analytics/event_publisher.py +++ b/localstack/utils/analytics/event_publisher.py @@ -93,7 +93,7 @@ def get_config_file_homedir(): def get_config_file_tempdir(): - return _get_config_file(os.path.join(config.TMP_FOLDER, ".localstack")) + return _get_config_file(os.path.join(config.dirs.tmp, ".localstack")) def get_machine_id(): diff --git a/localstack/utils/analytics/metadata.py b/localstack/utils/analytics/metadata.py index 0c2c9efbb5563..793830d93fec7 100644 --- a/localstack/utils/analytics/metadata.py +++ b/localstack/utils/analytics/metadata.py @@ -117,7 +117,7 @@ def get_config_file_homedir(): def get_config_file_tempdir(): - return _get_config_file(os.path.join(config.TMP_FOLDER, ".localstack")) + return _get_config_file(os.path.join(config.dirs.tmp, ".localstack")) def read_api_key_safe(): diff --git a/localstack/utils/bootstrap.py b/localstack/utils/bootstrap.py index 43da71e16b855..adc4b115edaab 100644 --- a/localstack/utils/bootstrap.py +++ b/localstack/utils/bootstrap.py @@ -9,6 +9,7 @@ from typing import Iterable, List, Optional, Set from localstack import config, constants +from localstack.config import Directories from localstack.runtime import hooks from localstack.utils.common import FileListener, chmod_r, mkdir, poll_condition from localstack.utils.docker_utils import ( @@ -468,7 +469,7 @@ def __init__(self, name: str = None): self.env_vars = dict() self.additional_flags = list() - self.logfile = os.path.join(config.TMP_FOLDER, f"{self.name}_container.log") + self.logfile = os.path.join(config.dirs.tmp, f"{self.name}_container.log") def _get_mount_volumes(self) -> List[SimpleVolumeBind]: # FIXME: VolumeMappings should be supported by the docker client @@ -567,18 +568,18 @@ def prepare_docker_start(): if DOCKER_CLIENT.is_container_running(container_name): raise ContainerExists('LocalStack container named "%s" is already running' % container_name) - if config.TMP_FOLDER != config.HOST_TMP_FOLDER and not config.LAMBDA_REMOTE_DOCKER: + if config.dirs.tmp != config.dirs.functions and not config.LAMBDA_REMOTE_DOCKER: print( - f"WARNING: The detected temp folder for localstack ({config.TMP_FOLDER}) is not equal to the " - f"HOST_TMP_FOLDER environment variable set ({config.HOST_TMP_FOLDER})." + f"WARNING: The detected temp folder for localstack ({config.dirs.tmp}) is not equal to the " + f"HOST_TMP_FOLDER environment variable set ({config.dirs.functions})." ) # Logger is not initialized at this point, so the warning is displayed via print os.environ[ENV_SCRIPT_STARTING_DOCKER] = "1" # make sure temp folder exists - mkdir(config.TMP_FOLDER) + mkdir(config.dirs.tmp) try: - chmod_r(config.TMP_FOLDER, 0o777) + chmod_r(config.dirs.tmp, 0o777) except Exception: pass @@ -613,27 +614,43 @@ def configure_container(container: LocalstackContainer): if value is not None: container.env_vars[env_var] = value container.env_vars["DOCKER_HOST"] = f"unix://{config.DOCKER_SOCK}" - container.env_vars["HOST_TMP_FOLDER"] = config.HOST_TMP_FOLDER + container.env_vars["HOST_TMP_FOLDER"] = config.dirs.functions # TODO: rename env var # TODO discuss if this should be the default? # to activate proper signal handling container.env_vars["SET_TERM_HANDLER"] = "1" - # data_dir mounting and environment variables - bind_mounts = [] - data_dir = os.environ.get("DATA_DIR", None) - if data_dir is not None: - container_data_dir = "/tmp/localstack_data" - container.volumes.add((data_dir, container_data_dir)) - container.env_vars["DATA_DIR"] = container_data_dir + configure_volume_mounts(container) - # default bind mounts - container.volumes.add((config.TMP_FOLDER, "/tmp/localstack")) - bind_mounts.append((config.DOCKER_SOCK, config.DOCKER_SOCK)) + # mount docker socket + container.volumes.append((config.DOCKER_SOCK, config.DOCKER_SOCK)) container.additional_flags.append("--privileged") +def configure_volume_mounts(container: LocalstackContainer): + source_dirs = config.dirs + target_dirs = Directories.for_container() + + # default shared directories + for name in Directories.default_bind_mounts: + src = getattr(source_dirs, name, None) + target = getattr(target_dirs, name, None) + if src and target: + container.volumes.add(VolumeBind(src, target)) + + # shared tmp folder + container.volumes.add(VolumeBind(source_dirs.tmp, target_dirs.tmp)) + + # data_dir mounting and environment variables + if source_dirs.data: + container.volumes.add(VolumeBind(source_dirs.data, target_dirs.data)) + container.env_vars["DATA_DIR"] = target_dirs.data + + if source_dirs.init: + container.volumes.add(VolumeBind(source_dirs.init, target_dirs.init)) + + @log_duration() def prepare_host(): """ diff --git a/localstack/utils/docker_utils.py b/localstack/utils/docker_utils.py index 1518499cd0dc8..9162e548ec21a 100644 --- a/localstack/utils/docker_utils.py +++ b/localstack/utils/docker_utils.py @@ -949,7 +949,7 @@ def rm_env_vars_file(env_vars_file) -> None: @staticmethod def mountable_tmp_file(): - f = os.path.join(config.TMP_FOLDER, short_uid()) + f = os.path.join(config.dirs.tmp, short_uid()) TMP_FILES.append(f) return f diff --git a/localstack/utils/persistence.py b/localstack/utils/persistence.py index 6f7d1cf28e4f6..c3607a4d107cf 100644 --- a/localstack/utils/persistence.py +++ b/localstack/utils/persistence.py @@ -12,7 +12,7 @@ from six import add_metaclass from localstack import config, constants -from localstack.config import DATA_DIR, is_env_not_false, is_env_true +from localstack.config import is_env_not_false, is_env_true from localstack.services.generic_proxy import ProxyListener from localstack.utils.aws import aws_stack from localstack.utils.bootstrap import is_api_enabled @@ -202,7 +202,7 @@ def restore_persisted_data(apis): def is_persistence_enabled(): - return bool(config.DATA_DIR) + return bool(config.dirs.data) def is_persistence_restored(): @@ -219,7 +219,7 @@ class StartupInfo(NamedTuple): def save_startup_info(): from localstack_ext.constants import VERSION as LOCALSTACK_EXT_VERSION - file_path = os.path.join(DATA_DIR, STARTUP_INFO_FILE) + file_path = os.path.join(config.dirs.data, STARTUP_INFO_FILE) info = StartupInfo( timestamp=datetime.datetime.now().isoformat(), @@ -257,9 +257,9 @@ def _append_startup_info(file_path, startup_info: StartupInfo): def get_file_path(api, create=True): if api not in API_FILE_PATHS: API_FILE_PATHS[api] = False - if not DATA_DIR: + if not config.dirs.data: return False - file_path = API_FILE_PATTERN.format(data_dir=DATA_DIR, api=api) + file_path = API_FILE_PATTERN.format(data_dir=config.dirs.data, api=api) if create and not os.path.exists(file_path): with open(file_path, "a"): os.utime(file_path, None) diff --git a/tests/bootstrap/test_cli.py b/tests/bootstrap/test_cli.py index 2384185349e76..38f24d0cf60db 100644 --- a/tests/bootstrap/test_cli.py +++ b/tests/bootstrap/test_cli.py @@ -4,7 +4,7 @@ from localstack import config, constants from localstack.cli.localstack import localstack as cli -from localstack.config import get_edge_url, in_docker +from localstack.config import DOCKER_SOCK, get_edge_url, in_docker from localstack.utils import docker_utils from localstack.utils.common import poll_condition @@ -109,3 +109,25 @@ def test_custom_docker_flags(self, runner, tmp_path, monkeypatch, container_clie inspect = container_client.inspect_container(config.MAIN_CONTAINER_NAME) assert "42069/tcp" in inspect["HostConfig"]["PortBindings"] assert f"{volume}:{volume}" in inspect["HostConfig"]["Binds"] + + def test_directories_mounted_correctly(self, runner, tmp_path, monkeypatch, container_client): + data_dir = tmp_path / "data_dir" + tmp_folder = tmp_path / "tmp" + + # set different directories and make sure they are mounted correctly + monkeypatch.setenv("DATA_DIR", str(data_dir)) + monkeypatch.setattr(config, "DATA_DIR", str(data_dir)) + monkeypatch.setattr(config, "TMP_FOLDER", str(tmp_folder)) + # reload directories from manipulated config + monkeypatch.setattr(config, "dirs", config.Directories.from_config()) + + runner.invoke(cli, ["start", "-d"]) + runner.invoke(cli, ["wait", "-t", "60"]) + + # check that mounts were created correctly + inspect = container_client.inspect_container(config.MAIN_CONTAINER_NAME) + container_dirs = config.Directories.for_container() + binds = inspect["HostConfig"]["Binds"] + assert f"{tmp_folder}:{container_dirs.tmp}" in binds + assert f"{data_dir}:{container_dirs.data}" in binds + assert f"{DOCKER_SOCK}:{DOCKER_SOCK}" in binds diff --git a/tests/integration/test_ses.py b/tests/integration/test_ses.py index b6414bfb1deb4..ddfd4c830d412 100644 --- a/tests/integration/test_ses.py +++ b/tests/integration/test_ses.py @@ -68,7 +68,7 @@ def test_get_identity_verification_attributes(self, ses_client): assert "VerificationToken" not in response[email] def test_send_email_save(self, ses_client): - data_dir = config.DATA_DIR or config.TMP_FOLDER + data_dir = config.dirs.data or config.dirs.tmp email = "user@example.com" ses_client.verify_email_address(EmailAddress=email) message = ses_client.send_email( @@ -99,7 +99,7 @@ def test_send_email_save(self, ses_client): assert ["success@example.com"] == contents["Destinations"]["ToAddresses"] def test_send_templated_email_save(self, ses_client, create_template): - data_dir = config.DATA_DIR or config.TMP_FOLDER + data_dir = config.dirs.data or config.dirs.tmp email = "user@example.com" ses_client.verify_email_address(EmailAddress=email) ses_client.delete_template(TemplateName=TEST_TEMPLATE_ATTRIBUTES["TemplateName"]) diff --git a/tests/unit/cli/test_lpm.py b/tests/unit/cli/test_lpm.py index 2a87545d63328..9cf2a76792433 100644 --- a/tests/unit/cli/test_lpm.py +++ b/tests/unit/cli/test_lpm.py @@ -3,7 +3,7 @@ import pytest from click.testing import CliRunner -from localstack.cli.lpm import cli +from localstack.cli.lpm import cli, console @pytest.fixture @@ -12,7 +12,7 @@ def runner(): def test_list(runner, monkeypatch): - monkeypatch.setenv("NO_COLOR", "1") + monkeypatch.setattr(console, "no_color", True) result = runner.invoke(cli, ["list"]) assert result.exit_code == 0 diff --git a/tests/unit/utils/test_persistence.py b/tests/unit/utils/test_persistence.py index 65115c2f8039e..3a06af7fd3dbb 100644 --- a/tests/unit/utils/test_persistence.py +++ b/tests/unit/utils/test_persistence.py @@ -1,21 +1,19 @@ import json -import unittest -import pytest from localstack_ext import constants as ext_constants -from localstack import constants -from localstack.utils import persistence -from localstack.utils.persistence import StartupInfo, _append_startup_info, save_startup_info +from localstack import config, constants +from localstack.utils.persistence import ( + STARTUP_INFO_FILE, + StartupInfo, + _append_startup_info, + save_startup_info, +) -class TestStartupInfo(unittest.TestCase): - @pytest.fixture(autouse=True) - def init_tmpdir(self, tmpdir): - self.tmpdir = tmpdir - - def test_append_startup_info_to_new_file(self): - file_path = self.tmpdir.join("testfile_1.json") +class TestStartupInfo: + def test_append_startup_info_to_new_file(self, tmp_path): + file_path = tmp_path / "testfile_1.json" _append_startup_info( file_path, StartupInfo("2021-07-07T14:25:36.0", "1.0.0", "0.1.0", False) @@ -32,8 +30,8 @@ def test_append_startup_info_to_new_file(self): assert d["localstack_ext_version"] == "0.1.0" assert d["pro_activated"] is False - def test_append_startup_info_maintains_order(self): - file_path = self.tmpdir.join("testfile_2.json") + def test_append_startup_info_maintains_order(self, tmp_path): + file_path = tmp_path / "testfile_2.json" _append_startup_info( file_path, StartupInfo("2021-07-07T14:25:36.0", "1.0.0", "0.1.0", False) @@ -50,24 +48,22 @@ def test_append_startup_info_maintains_order(self): d = doc[1] assert d["timestamp"] == "2021-07-13T11:48:15.1" - def test_save_startup_info(self): - old_data_dir = persistence.DATA_DIR - try: - persistence.DATA_DIR = self.tmpdir - save_startup_info() + def test_save_startup_info(self, tmp_path, monkeypatch): + data_dir = tmp_path / "data" + monkeypatch.setattr(config.dirs, "data", data_dir) + config.dirs.mkdirs() - file_path = self.tmpdir.join(persistence.STARTUP_INFO_FILE) + save_startup_info() - with file_path.open("r") as fd: - doc = json.load(fd) + file_path = data_dir / STARTUP_INFO_FILE - assert len(doc) == 1 - d = doc[0] + with file_path.open("r") as fd: + doc = json.load(fd) - assert d["timestamp"] - assert d["localstack_version"] == constants.VERSION - assert d["localstack_ext_version"] == ext_constants.VERSION - assert d["pro_activated"] in [False, True] + assert len(doc) == 1 + d = doc[0] - finally: - persistence.DATA_DIR = old_data_dir + assert d["timestamp"] + assert d["localstack_version"] == constants.VERSION + assert d["localstack_ext_version"] == ext_constants.VERSION + assert d["pro_activated"] in [False, True]