diff --git a/.gitignore b/.gitignore index df90169c4d544..93112312b4f80 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ .coverage.* htmlcov +.cache +.filesystem /infra/ localstack/infra/ diff --git a/Dockerfile b/Dockerfile index 557d60781a952..ccf3695d0a689 100644 --- a/Dockerfile +++ b/Dockerfile @@ -90,6 +90,16 @@ ADD https://raw.githubusercontent.com/carlossg/docker-maven/master/openjdk-11/se RUN mkdir -p /opt/code/localstack WORKDIR /opt/code/localstack/ +# create filesystem hierarchy +RUN mkdir -p /var/lib/localstack && \ + mkdir -p /usr/lib/localstack +# backwards compatibility with LEGACY_DIRECTORIES (TODO: deprecate and remove) +RUN mkdir -p /opt/code/localstack/localstack && \ + ln -s /usr/lib/localstack /opt/code/localstack/localstack/infra && \ + mkdir /tmp/localstack && \ + chmod -R 777 /tmp/localstack && \ + chmod -R 777 /usr/lib/localstack + # install basic (global) tools to final image RUN pip install --no-cache-dir --upgrade supervisor virtualenv @@ -130,12 +140,9 @@ RUN (cd /tmp && git clone https://github.com/timescale/timescaledb.git) && \ # init environment and cache some dependencies ARG DYNAMODB_ZIP_URL=https://s3-us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_latest.zip -RUN mkdir -p /opt/code/localstack/localstack/infra/dynamodb && \ +RUN mkdir -p /usr/lib/localstack/dynamodb && \ curl -L -o /tmp/localstack.ddb.zip ${DYNAMODB_ZIP_URL} && \ - (cd localstack/infra/dynamodb && unzip -q /tmp/localstack.ddb.zip && rm /tmp/localstack.ddb.zip) && \ - mkdir -p /opt/code/localstack/localstack/infra/elasticmq && \ - curl -L -o /opt/code/localstack/localstack/infra/elasticmq/elasticmq-server.jar \ - https://s3-eu-west-1.amazonaws.com/softwaremill-public/elasticmq-server-1.1.0.jar + (cd /usr/lib/localstack/dynamodb && unzip -q /tmp/localstack.ddb.zip && rm /tmp/localstack.ddb.zip) # upgrade python build tools RUN (virtualenv .venv && source .venv/bin/activate && pip3 install --upgrade pip wheel setuptools) @@ -158,8 +165,7 @@ RUN (virtualenv .venv && source .venv/bin/activate && pip3 uninstall -y localsta # base-light: Stage which does not add additional dependencies (like elasticsearch) FROM base as base-light -RUN mkdir -p /opt/code/localstack/localstack/infra && \ - touch localstack/infra/.light-version +RUN touch /usr/lib/localstack/.light-version @@ -170,13 +176,12 @@ FROM base as base-full # https://github.com/pires/docker-elasticsearch/issues/56 ENV ES_TMPDIR /tmp -ENV ES_BASE_DIR=localstack/infra/elasticsearch +ENV ES_BASE_DIR=/usr/lib/localstack/elasticsearch ENV ES_JAVA_HOME /usr/lib/jvm/java-11 RUN TARGETARCH_SYNONYM=$([[ "$TARGETARCH" == "amd64" ]] && echo "x86_64" || echo "aarch64"); \ - mkdir -p /opt/code/localstack/localstack/infra && \ curl -L -o /tmp/localstack.es.tar.gz \ https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.10.0-linux-${TARGETARCH_SYNONYM}.tar.gz && \ - (cd localstack/infra/ && tar -xf /tmp/localstack.es.tar.gz && \ + (cd /usr/lib/localstack && tar -xf /tmp/localstack.es.tar.gz && \ mv elasticsearch* elasticsearch && rm /tmp/localstack.es.tar.gz) && \ (cd $ES_BASE_DIR && \ bin/elasticsearch-plugin install analysis-icu && \ @@ -213,24 +218,24 @@ COPY --from=builder /opt/code/localstack/ /opt/code/localstack/ COPY --from=builder /usr/share/postgresql/11/extension /usr/share/postgresql/11/extension COPY --from=builder /usr/lib/postgresql/11/lib /usr/lib/postgresql/11/lib -RUN mkdir -p /tmp/localstack && \ - if [ -e /usr/bin/aws ]; then mv /usr/bin/aws /usr/bin/aws.bk; fi; ln -s /opt/code/localstack/.venv/bin/aws /usr/bin/aws +RUN if [ -e /usr/bin/aws ]; then mv /usr/bin/aws /usr/bin/aws.bk; fi; ln -s /opt/code/localstack/.venv/bin/aws /usr/bin/aws # fix some permissions and create local user RUN mkdir -p /.npm && \ chmod 777 . && \ chmod 755 /root && \ chmod -R 777 /.npm && \ - chmod -R 777 /tmp/localstack && \ + chmod -R 777 /var/lib/localstack && \ useradd -ms /bin/bash localstack && \ ln -s `pwd` /tmp/localstack_install_dir # Install the latest version of awslocal globally RUN pip3 install --upgrade awscli awscli-local requests -# Add the code in the last step -# Also adds the results of `make init` to the container. +# Adds the results of `make init` to the container. # `make init` _needs_ to be executed before building this docker image (since the execution needs docker itself). +ADD .filesystem/usr/lib/localstack /usr/lib/localstack +# Add the code in the last step ADD localstack/ localstack/ # Download some more dependencies (make init needs the LocalStack code) @@ -259,5 +264,8 @@ EXPOSE 4566 4510-4559 5678 HEALTHCHECK --interval=10s --start-period=15s --retries=5 --timeout=5s CMD ./bin/localstack status services --format=json +# default volume directory +VOLUME /var/lib/localstack + # define command at startup ENTRYPOINT ["docker-entrypoint.sh"] diff --git a/Makefile b/Makefile index 4a1c4d0533fc0..55d703fc52d10 100644 --- a/Makefile +++ b/Makefile @@ -193,7 +193,7 @@ docker-run: ## Run Docker image locally docker-mount-run: MOTO_DIR=$$(echo $$(pwd)/.venv/lib/python*/site-packages/moto | awk '{print $$NF}'); echo MOTO_DIR $$MOTO_DIR; \ - DOCKER_FLAGS="$(DOCKER_FLAGS) -v `pwd`/localstack/constants.py:/opt/code/localstack/localstack/constants.py -v `pwd`/localstack/config.py:/opt/code/localstack/localstack/config.py -v `pwd`/localstack/plugins.py:/opt/code/localstack/localstack/plugins.py -v `pwd`/localstack/plugin:/opt/code/localstack/localstack/plugin -v `pwd`/localstack/runtime:/opt/code/localstack/localstack/runtime -v `pwd`/localstack/utils:/opt/code/localstack/localstack/utils -v `pwd`/localstack/services:/opt/code/localstack/localstack/services -v `pwd`/localstack/http:/opt/code/localstack/localstack/http -v `pwd`/localstack/contrib:/opt/code/localstack/localstack/contrib -v `pwd`/localstack/dashboard:/opt/code/localstack/localstack/dashboard -v `pwd`/tests:/opt/code/localstack/tests -v $$MOTO_DIR:/opt/code/localstack/.venv/lib/python3.8/site-packages/moto/" make docker-run + DOCKER_FLAGS="$(DOCKER_FLAGS) -v `pwd`/localstack/constants.py:/opt/code/localstack/localstack/constants.py -v `pwd`/localstack/config.py:/opt/code/localstack/localstack/config.py -v `pwd`/localstack/plugins.py:/opt/code/localstack/localstack/plugins.py -v `pwd`/localstack/plugin:/opt/code/localstack/localstack/plugin -v `pwd`/localstack/runtime:/opt/code/localstack/localstack/runtime -v `pwd`/localstack/utils:/opt/code/localstack/localstack/utils -v `pwd`/localstack/services:/opt/code/localstack/localstack/services -v `pwd`/localstack/http:/opt/code/localstack/localstack/http -v `pwd`/localstack/contrib:/opt/code/localstack/localstack/contrib -v `pwd`/localstack/dashboard:/opt/code/localstack/localstack/dashboard -v `pwd`/tests:/opt/code/localstack/tests -v $$MOTO_DIR:/opt/code/localstack/.venv/lib/python3.10/site-packages/moto/" make docker-run docker-build-lambdas: docker build -t localstack/lambda-js:nodejs14.x -f bin/lambda/Dockerfile.nodejs14x . @@ -220,13 +220,13 @@ test-docker-mount: ## Run automated tests in Docker (mounting local code) # TODO: find a cleaner way to mount/copy the dependencies into the container... VENV_DIR=$$(pwd)/.venv/; \ PKG_DIR=$$(echo $$VENV_DIR/lib/python*/site-packages | awk '{print $$NF}'); \ - PKG_DIR_CON=/opt/code/localstack/.venv/lib/python3.8/site-packages; \ + PKG_DIR_CON=/opt/code/localstack/.venv/lib/python3.10/site-packages; \ echo "#!/usr/bin/env python" > /tmp/pytest.ls.bin; cat $$VENV_DIR/bin/pytest >> /tmp/pytest.ls.bin; chmod +x /tmp/pytest.ls.bin; \ - DOCKER_FLAGS="-v `pwd`/tests:/opt/code/localstack/tests -v /tmp/pytest.ls.bin:/opt/code/localstack/.venv/bin/pytest -v $$PKG_DIR/py:$$PKG_DIR_CON/py -v $$PKG_DIR/pluggy:$$PKG_DIR_CON/pluggy -v $$PKG_DIR/iniconfig:$$PKG_DIR_CON/iniconfig -v $$PKG_DIR/packaging:$$PKG_DIR_CON/packaging -v $$PKG_DIR/pytest:$$PKG_DIR_CON/pytest -v $$PKG_DIR/_pytest:/opt/code/localstack/.venv/lib/python3.8/site-packages/_pytest" make test-docker-mount-code + DOCKER_FLAGS="-v `pwd`/tests:/opt/code/localstack/tests -v /tmp/pytest.ls.bin:/opt/code/localstack/.venv/bin/pytest -v $$PKG_DIR/py:$$PKG_DIR_CON/py -v $$PKG_DIR/pluggy:$$PKG_DIR_CON/pluggy -v $$PKG_DIR/iniconfig:$$PKG_DIR_CON/iniconfig -v $$PKG_DIR/packaging:$$PKG_DIR_CON/packaging -v $$PKG_DIR/pytest:$$PKG_DIR_CON/pytest -v $$PKG_DIR/_pytest:/opt/code/localstack/.venv/lib/python3.10/site-packages/_pytest" make test-docker-mount-code test-docker-mount-code: PACKAGES_DIR=$$(echo $$(pwd)/.venv/lib/python*/site-packages | awk '{print $$NF}'); \ - DOCKER_FLAGS="$(DOCKER_FLAGS) --entrypoint= -v `pwd`/localstack/config.py:/opt/code/localstack/localstack/config.py -v `pwd`/localstack/constants.py:/opt/code/localstack/localstack/constants.py -v `pwd`/localstack/utils:/opt/code/localstack/localstack/utils -v `pwd`/localstack/services:/opt/code/localstack/localstack/services -v `pwd`/localstack/aws:/opt/code/localstack/localstack/aws -v `pwd`/Makefile:/opt/code/localstack/Makefile -v $$PACKAGES_DIR/moto:/opt/code/localstack/.venv/lib/python3.8/site-packages/moto/ -e TEST_PATH=\\'$(TEST_PATH)\\' -e LAMBDA_JAVA_OPTS=$(LAMBDA_JAVA_OPTS) $(ENTRYPOINT)" CMD="make test" make docker-run + DOCKER_FLAGS="$(DOCKER_FLAGS) --entrypoint= -v `pwd`/localstack/config.py:/opt/code/localstack/localstack/config.py -v `pwd`/localstack/constants.py:/opt/code/localstack/localstack/constants.py -v `pwd`/localstack/utils:/opt/code/localstack/localstack/utils -v `pwd`/localstack/services:/opt/code/localstack/localstack/services -v `pwd`/localstack/aws:/opt/code/localstack/localstack/aws -v `pwd`/Makefile:/opt/code/localstack/Makefile -v $$PACKAGES_DIR/moto:/opt/code/localstack/.venv/lib/python3.10/site-packages/moto/ -e TEST_PATH=\\'$(TEST_PATH)\\' -e LAMBDA_JAVA_OPTS=$(LAMBDA_JAVA_OPTS) $(ENTRYPOINT)" CMD="make test" make docker-run # Note: the ci-* targets below should only be used in CI builds! @@ -277,7 +277,9 @@ init-precommit: ## install te pre-commit hook into your local git reposit ($(VENV_RUN); pre-commit install) clean: ## Clean up (npm dependencies, downloaded infrastructure code, compiled Java classes) + rm -rf .filesystem rm -rf localstack/dashboard/web/node_modules + # TODO: remove localstack/infra/ as it's no longer used rm -rf localstack/infra/amazon-kinesis-client rm -rf localstack/infra/elasticsearch rm -rf localstack/infra/elasticmq diff --git a/bin/docker-entrypoint.sh b/bin/docker-entrypoint.sh index de250e77fdab9..de7c69085a270 100755 --- a/bin/docker-entrypoint.sh +++ b/bin/docker-entrypoint.sh @@ -5,7 +5,7 @@ shopt -s nullglob if [[ ! $INIT_SCRIPTS_PATH ]] then - # FIXME: move + # FIXME: move to /etc/localstack/init/ready.d INIT_SCRIPTS_PATH=/docker-entrypoint-initaws.d fi if [[ ! $EDGE_PORT ]] @@ -45,14 +45,21 @@ if [ "$DISABLE_TERM_HANDLER" == "" ]; then trap 'kill -31 ${!}; term_handler 31' SIGUSR2 fi -cat /dev/null > /tmp/localstack_infra.log -cat /dev/null > /tmp/localstack_infra.err +LOG_DIR=/var/lib/localstack/logs +test -d ${LOG_DIR} || mkdir -p ${LOG_DIR} + +cat /dev/null > ${LOG_DIR}/localstack_infra.log +cat /dev/null > ${LOG_DIR}/localstack_infra.err + +# for backwards compatibility with LEGACY_DIRECTORIES=1 +test -f /tmp/localstack_infra.log || ln -s ${LOG_DIR}/localstack_infra.log /tmp/localstack_infra.log +test -f /tmp/localstack_infra.err || ln -s ${LOG_DIR}/localstack_infra.err /tmp/localstack_infra.err supervisord -c /etc/supervisord.conf & suppid="$!" function run_startup_scripts { - until grep -q '^Ready.' /tmp/localstack_infra.log >/dev/null 2>&1 ; do + until grep -q '^Ready.' ${LOG_DIR}/localstack_infra.log >/dev/null 2>&1 ; do echo "Waiting for all LocalStack services to be ready" sleep 7 done @@ -73,8 +80,8 @@ run_startup_scripts & # Run tail on the localstack log files forever until we are told to terminate if [ "$DISABLE_TERM_HANDLER" == "" ]; then while true; do - tail -qF /tmp/localstack_infra.log /tmp/localstack_infra.err & wait ${!} + tail -qF ${LOG_DIR}/localstack_infra.log ${LOG_DIR}/localstack_infra.err & wait ${!} done else - tail -qF /tmp/localstack_infra.log /tmp/localstack_infra.err + tail -qF ${LOG_DIR}/localstack_infra.log ${LOG_DIR}/localstack_infra.err fi diff --git a/bin/supervisord.conf b/bin/supervisord.conf index 5c671b7b43757..1d2c4d5ba2931 100644 --- a/bin/supervisord.conf +++ b/bin/supervisord.conf @@ -9,5 +9,5 @@ environment= LOCALSTACK_INFRA_PROCESS=1 autostart=true autorestart=true -stdout_logfile=/tmp/localstack_infra.log -stderr_logfile=/tmp/localstack_infra.err +stdout_logfile=/var/lib/localstack/logs/localstack_infra.log +stderr_logfile=/var/lib/localstack/logs/localstack_infra.err diff --git a/localstack/cli/lpm.py b/localstack/cli/lpm.py index 342e10c06bc20..3edad5fd0407e 100644 --- a/localstack/cli/lpm.py +++ b/localstack/cli/lpm.py @@ -5,6 +5,7 @@ from click import ClickException from rich.console import Console +from localstack import config from localstack.services.install import InstallerManager from localstack.utils.bootstrap import setup_logging @@ -59,6 +60,7 @@ def install(package, parallel): """ console.print(f"resolving packages: {package}") installers: Dict[str, Callable] = InstallerManager().get_installers() + config.dirs.mkdirs() for pkg in package: if pkg not in installers: diff --git a/localstack/config.py b/localstack/config.py index aba1919f8ade5..f694f40ec66b4 100644 --- a/localstack/config.py +++ b/localstack/config.py @@ -15,12 +15,14 @@ DEFAULT_LAMBDA_CONTAINER_REGISTRY, DEFAULT_PORT_EDGE, DEFAULT_SERVICE_PORTS, + DEFAULT_VOLUME_DIR, ENV_INTERNAL_TEST_RUN, FALSE_STRINGS, - INSTALL_DIR_INFRA, LOCALHOST, LOCALHOST_IP, + LOCALSTACK_ROOT_FOLDER, LOG_LEVELS, + MODULE_MAIN_PATH, TRACE_LOG_LEVELS, TRUE_STRINGS, ) @@ -61,15 +63,15 @@ class Directories: def __init__( self, - static_libs: str = None, - var_libs: str = None, - cache: str = None, - tmp: str = None, - functions: str = None, - data: str = None, - config: str = None, - init: str = None, - logs: str = None, + static_libs: str, + var_libs: str, + cache: str, + tmp: str, + functions: str, + data: str, + config: str, + init: str, + logs: str, ) -> None: super().__init__() self.static_libs = static_libs @@ -83,10 +85,81 @@ def __init__( self.logs = logs @staticmethod - def from_config(): + def defaults() -> "Directories": + """Returns Localstack directory paths based on the localstack filesystem hierarchy.""" + return Directories( + static_libs="/usr/lib/localstack", + var_libs=f"{DEFAULT_VOLUME_DIR}/lib", + cache=f"{DEFAULT_VOLUME_DIR}/cache", + tmp=f"{DEFAULT_VOLUME_DIR}/tmp", + functions=f"{DEFAULT_VOLUME_DIR}/tmp", # FIXME: remove - this was misconceived + data=f"{DEFAULT_VOLUME_DIR}/state", + logs=f"{DEFAULT_VOLUME_DIR}/logs", + config="/etc/localstack/conf.d", # for future use + init="/etc/localstack/init", # for future use + ) + + @staticmethod + def for_container() -> "Directories": + """ + Returns Localstack directory paths as they are defined within the container. Everything shared and writable + lives in /var/lib/localstack or /tmp/localstack. + + :returns: Directories object + """ + defaults = Directories.defaults() + + return Directories( + static_libs=defaults.static_libs, + var_libs=defaults.var_libs, + cache=defaults.cache, + tmp=defaults.tmp, + functions=defaults.functions, + data=defaults.data if PERSISTENCE else os.path.join(defaults.tmp, "state"), + config=defaults.config, + logs=defaults.logs, + init="/docker-entrypoint-initaws.d", # FIXME should be reworked with lifecycle hooks + ) + + @staticmethod + def for_host() -> "Directories": + """Return directories used for running localstack in host mode. Note that these are *not* the directories + that are mounted into the container when the user starts localstack.""" + root = os.environ.get("FILESYSTEM_ROOT") or os.path.join( + LOCALSTACK_ROOT_FOLDER, ".filesystem" + ) + root = os.path.abspath(root) + + defaults = Directories.for_container() + + tmp = os.path.join(root, defaults.tmp.lstrip("/")) + data = os.path.join(root, defaults.data.lstrip("/")) + + return Directories( + static_libs=os.path.join(root, defaults.static_libs.lstrip("/")), + var_libs=os.path.join(root, defaults.var_libs.lstrip("/")), + cache=os.path.join(root, defaults.cache.lstrip("/")), + tmp=tmp, + functions=os.path.join(root, defaults.functions.lstrip("/")), + data=data if PERSISTENCE else os.path.join(tmp, "state"), + config=os.path.join(root, defaults.config.lstrip("/")), + init=os.path.join(root, defaults.init.lstrip("/")), + logs=os.path.join(root, defaults.logs.lstrip("/")), + ) + + @staticmethod + def legacy_from_config(): """Returns Localstack directory paths from the config/environment variables defined by the config.""" # Note that the entries should be unique, as further downstream in docker_utils.py we're removing # duplicate host paths in the volume mounts via `dict(mount_volumes)`. + + # legacy config variables inlined + INSTALL_DIR_INFRA = os.path.join(MODULE_MAIN_PATH, "infra") + # ephemeral cache dir that persists across reboots + CACHE_DIR = os.environ.get("CACHE_DIR", os.path.join(TMP_FOLDER, "cache")).strip() + # libs cache dir that persists across reboots + VAR_LIBS_DIR = os.environ.get("VAR_LIBS_DIR", os.path.join(TMP_FOLDER, "var_libs")).strip() + return Directories( static_libs=INSTALL_DIR_INFRA, var_libs=VAR_LIBS_DIR, @@ -100,7 +173,7 @@ def from_config(): ) @staticmethod - def for_container() -> "Directories": + def legacy_for_container() -> "Directories": """ Returns Localstack directory paths as they are defined within the container. Everything shared and writable lives in /var/lib/localstack or /tmp/localstack. @@ -110,10 +183,9 @@ def for_container() -> "Directories": # only set CONTAINER_VAR_LIBS_FOLDER/CONTAINER_CACHE_FOLDER inside the container to redirect var_libs/cache to # another directory to avoid override by host mount var_libs = ( - os.environ.get("CONTAINER_VAR_LIBS_FOLDER", "").strip() - or "/var/lib/localstack/var_libs" + os.environ.get("CONTAINER_VAR_LIBS_FOLDER", "").strip() or "/tmp/localstack/var_libs" ) - cache = os.environ.get("CONTAINER_CACHE_FOLDER", "").strip() or "/var/lib/localstack/cache" + cache = os.environ.get("CONTAINER_CACHE_FOLDER", "").strip() or "/tmp/localstack/cache" tmp = ( os.environ.get("CONTAINER_TMP_FOLDER", "").strip() or "/tmp/localstack" ) # TODO: discuss movement to /var/lib/localstack/tmp @@ -121,14 +193,14 @@ def for_container() -> "Directories": DATA_DIR if in_docker() else "/tmp/localstack_data" ) # TODO: move to /var/lib/localstack/data return Directories( - static_libs=INSTALL_DIR_INFRA, + static_libs=os.path.join(MODULE_MAIN_PATH, "infra"), var_libs=var_libs, cache=cache, tmp=tmp, functions=HOST_TMP_FOLDER, # TODO: move to /var/lib/localstack/tmp data=data_dir, config=None, # config directory is host-only - logs="/var/lib/localstack/logs", + logs="/tmp/localstack/logs", init="/docker-entrypoint-initaws.d", ) @@ -277,12 +349,21 @@ def in_docker(): # name of the host under which the LocalStack services are available LOCALSTACK_HOSTNAME = os.environ.get("LOCALSTACK_HOSTNAME", "").strip() or LOCALHOST -# directory for persisting data +# directory for persisting data (TODO: deprecated, simply use PERSISTENCE=1) DATA_DIR = os.environ.get("DATA_DIR", "").strip() +# whether localstack should persist service state across localstack runs +PERSISTENCE = is_env_true("PERSISTENCE") + +# whether to clear config.dirs.tmp on startup and shutdown +CLEAR_TMP_FOLDER = is_env_not_false("CLEAR_TMP_FOLDER") + # folder for temporary files and data TMP_FOLDER = os.path.join(tempfile.gettempdir(), "localstack") +# this is exclusively for the CLI to configure the container mount into /var/lib/localstack +VOLUME_DIR = os.environ.get("LOCALSTACK_VOLUME_DIR", "").strip() or TMP_FOLDER + # fix for Mac OS, to be able to mount /var/folders in Docker if TMP_FOLDER.startswith("/var/folders/") and os.path.exists("/private%s" % TMP_FOLDER): TMP_FOLDER = "/private%s" % TMP_FOLDER @@ -290,10 +371,8 @@ def in_docker(): # temporary folder of the host (required when running in Docker). Fall back to local tmp folder if not set HOST_TMP_FOLDER = os.environ.get("HOST_TMP_FOLDER", TMP_FOLDER) -# ephemeral cache dir that persists across reboots -CACHE_DIR = os.environ.get("CACHE_DIR", os.path.join(TMP_FOLDER, "cache")).strip() -# libs cache dir that persists across reboots -VAR_LIBS_DIR = os.environ.get("VAR_LIBS_DIR", os.path.join(TMP_FOLDER, "var_libs")).strip() +# whether to use the old directory structure and mounting config +LEGACY_DIRECTORIES = is_env_true("LEGACY_DIRECTORIES") # whether to enable verbose debug logging LS_LOG = eval_log_type("LS_LOG") @@ -448,7 +527,6 @@ def in_docker(): except socket.error: pass - # ----- # SERVICE-SPECIFIC CONFIGS BELOW # ----- @@ -669,6 +747,7 @@ def in_docker(): "LAMBDA_REMOVE_CONTAINERS", "LAMBDA_STAY_OPEN_MODE", "LAMBDA_TRUNCATE_STDOUT", + "LEGACY_DIRECTORIES", "LEGACY_DOCKER_CLIENT", "LEGACY_EDGE_PROXY", "LOCALSTACK_API_KEY", @@ -680,6 +759,7 @@ def in_docker(): "OPENSEARCH_ENDPOINT_STRATEGY", "OUTBOUND_HTTP_PROXY", "OUTBOUND_HTTPS_PROXY", + "PERSISTENCE", "PERSISTENCE_SINGLE_FILE", "REQUESTS_CA_BUNDLE", "S3_SKIP_SIGNATURE_VALIDATION", @@ -908,18 +988,58 @@ def __iter__(self): SERVICE_PROVIDER_CONFIG.load_from_environment() -# initialize directories -if is_in_docker: - dirs = Directories.for_container() -else: - dirs = Directories.from_config() -dirs.mkdirs() +def init_legacy_directories() -> Directories: + global PERSISTENCE + from localstack import constants -# TODO: remove deprecation warning with next release -for path in [dirs.config, os.path.join(dirs.tmp, ".localstack")]: - if path and os.path.isfile(path): - print( - f"warning: the config file .localstack is deprecated and no longer used, " - f"please remove it by running rm {path}" - ) + constants.DEFAULT_VOLUME_DIR = "/tmp/localstack" + + if DATA_DIR: + PERSISTENCE = True + + if is_in_docker: + dirs = Directories.legacy_for_container() + else: + dirs = Directories.legacy_from_config() + + dirs.mkdirs() + return dirs + + +def init_directories() -> Directories: + global DATA_DIR, PERSISTENCE + + if DATA_DIR: + # deprecation path: DATA_DIR being set means persistence is activated, but we're ignoring the path set in + # DATA_DIR + os.environ["PERSISTENCE"] = "1" + PERSISTENCE = True + + if is_in_docker: + dirs = Directories.for_container() + else: + dirs = Directories.for_host() + + if PERSISTENCE: + if DATA_DIR: + LOG.warning( + "Persistence mode was activated using the DATA_DIR variable. The DATA_DIR is deprecated and " + "its value is ignore. The data is instead stored into %s in the LocalStack filesystem.", + dirs.data, + ) + + # deprecation path + DATA_DIR = dirs.data + os.environ["DATA_DIR"] = dirs.data # still needed for external tools + + return dirs + + +# initialize directories +dirs: Directories +if LEGACY_DIRECTORIES: + CLEAR_TMP_FOLDER = False + dirs = init_legacy_directories() +else: + dirs = init_directories() diff --git a/localstack/constants.py b/localstack/constants.py index e1c3dbab56b6d..434feaf0e1a5a 100644 --- a/localstack/constants.py +++ b/localstack/constants.py @@ -50,9 +50,6 @@ MODULE_MAIN_PATH = os.path.dirname(os.path.realpath(__file__)) # TODO rename to "ROOT_FOLDER"! LOCALSTACK_ROOT_FOLDER = os.path.realpath(os.path.join(MODULE_MAIN_PATH, "..")) -INSTALL_DIR_INFRA = os.path.join( - MODULE_MAIN_PATH, "infra" -) # FIXME: deprecated, use config.dirs.infra # virtualenv folder LOCALSTACK_VENV_FOLDER = os.environ.get("VIRTUAL_ENV") @@ -65,6 +62,9 @@ os.path.join(LOCALSTACK_ROOT_FOLDER, "..", "..", "..") ) +# default volume directory containing shared data +DEFAULT_VOLUME_DIR = "/var/lib/localstack" + # API Gateway path to indicate a user request sent to the gateway PATH_USER_REQUEST = "_user_request_" diff --git a/localstack/services/awslambda/lambda_executors.py b/localstack/services/awslambda/lambda_executors.py index 6a17f57a9ddd0..382d86c6a571a 100644 --- a/localstack/services/awslambda/lambda_executors.py +++ b/localstack/services/awslambda/lambda_executors.py @@ -18,8 +18,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union from localstack import config -from localstack.config import LAMBDA_TRUNCATE_STDOUT, TMP_FOLDER -from localstack.constants import DEFAULT_LAMBDA_CONTAINER_REGISTRY +from localstack.constants import DEFAULT_LAMBDA_CONTAINER_REGISTRY, DEFAULT_VOLUME_DIR from localstack.runtime.hooks import hook_spec from localstack.services.awslambda.lambda_utils import ( API_PATH_ROOT, @@ -72,7 +71,7 @@ DockerContainerStatus, PortMappings, ) -from localstack.utils.docker_utils import DOCKER_CLIENT +from localstack.utils.docker_utils import DOCKER_CLIENT, get_default_volume_dir_mount from localstack.utils.run import FuncThread # constants @@ -1272,7 +1271,6 @@ def execute_in_container( class LambdaExecutorLocal(LambdaExecutor): - # maps functionARN -> functionVersion -> callable used to invoke a Lambda function locally FUNCTION_CALLABLES: Dict[str, Dict[str, Callable]] = {} @@ -1577,7 +1575,23 @@ def get_java_opts(cls): @classmethod def get_host_path_for_path_in_docker(cls, path): - return re.sub(r"^%s/(.*)$" % config.dirs.tmp, r"%s/\1" % config.dirs.functions, path) + if config.LEGACY_DIRECTORIES: + return re.sub(r"^%s/(.*)$" % config.dirs.tmp, r"%s/\1" % config.dirs.functions, path) + + if config.is_in_docker: + volume = get_default_volume_dir_mount() + + if volume: + if volume.type != "bind": + raise ValueError( + f"Mount to {DEFAULT_VOLUME_DIR} needs to be a bind mount for lambda code mounting to work" + ) + + return re.sub(r"^%s/(.*)$" % config.dirs.tmp, r"%s/\1" % volume.source, path) + else: + raise ValueError(f"No volume mounted to {DEFAULT_VOLUME_DIR}") + + return path @classmethod def format_windows_path(cls, path): @@ -1650,23 +1664,22 @@ def inject_endpoints_into_env(env_vars: Dict[str, str]): class OutputLog: - __slots__ = ["_stdout", "_stderr"] def __init__(self, stdout, stderr): self._stdout = stdout self._stderr = stderr - def stderr_formatted(self, truncated_to: int = LAMBDA_TRUNCATE_STDOUT): + def stderr_formatted(self, truncated_to: int = config.LAMBDA_TRUNCATE_STDOUT): return truncate(to_str(self._stderr).strip().replace("\n", "\n> "), truncated_to) - def stdout_formatted(self, truncated_to: int = LAMBDA_TRUNCATE_STDOUT): + def stdout_formatted(self, truncated_to: int = config.LAMBDA_TRUNCATE_STDOUT): return truncate(to_str(self._stdout).strip(), truncated_to) def output_file(self): try: with tempfile.NamedTemporaryFile( - dir=TMP_FOLDER, delete=False, suffix=".log", prefix="lambda_" + dir=config.dirs.tmp, delete=False, suffix=".log", prefix="lambda_" ) as f: LOG.info(f"writing log to file '{f.name}'") f.write(to_bytes(self._stderr)) diff --git a/localstack/services/infra.py b/localstack/services/infra.py index ec05d0ae32c26..e50a7c6c947a7 100644 --- a/localstack/services/infra.py +++ b/localstack/services/infra.py @@ -20,7 +20,7 @@ from localstack.services import generic_proxy, install, motoserver from localstack.services.generic_proxy import ProxyListener, start_proxy_server from localstack.services.plugins import SERVICE_PLUGINS, ServiceDisabled, wait_for_infra_shutdown -from localstack.utils import analytics, config_listener, persistence +from localstack.utils import analytics, config_listener, files, persistence from localstack.utils.analytics import event_publisher from localstack.utils.aws.request_context import patch_moto_request_handling from localstack.utils.bootstrap import canonicalize_api_names, in_ci, log_duration, setup_logging @@ -304,6 +304,9 @@ def cleanup_resources(): cleanup_tmp_files() cleanup_threads_and_processes() + if config.CLEAR_TMP_FOLDER: + files.rm_rf(config.dirs.tmp) + def log_startup_message(service): LOG.info("Starting mock %s service on %s ...", service, config.edge_ports_info()) @@ -367,6 +370,10 @@ def print_runtime_information(in_docker=False): def start_infra(asynchronous=False, apis=None): + if config.CLEAR_TMP_FOLDER: + files.rm_rf(config.dirs.tmp) # clear temp dir on startup + config.dirs.mkdirs() + events.infra_starting.set() try: @@ -499,7 +506,7 @@ def start_runtime_components(): thread = start_runtime_components() preload_services() - if config.dirs.data: + if config.PERSISTENCE: persistence.save_startup_info() print(READY_MARKER_OUTPUT) diff --git a/localstack/services/install.py b/localstack/services/install.py index e708dfec7d28e..a92be0df50177 100644 --- a/localstack/services/install.py +++ b/localstack/services/install.py @@ -60,7 +60,7 @@ INSTALL_DIR_KCL = "%s/amazon-kinesis-client" % dirs.static_libs INSTALL_DIR_STEPFUNCTIONS = "%s/stepfunctions" % dirs.static_libs INSTALL_DIR_KMS = "%s/kms" % dirs.static_libs -INSTALL_DIR_ELASTICMQ = "%s/elasticmq" % dirs.static_libs +INSTALL_DIR_ELASTICMQ = "%s/elasticmq" % dirs.var_libs INSTALL_DIR_KINESIS_MOCK = os.path.join(dirs.static_libs, "kinesis-mock") INSTALL_PATH_LOCALSTACK_FAT_JAR = "%s/localstack-utils-fat.jar" % dirs.static_libs INSTALL_PATH_DDB_JAR = os.path.join(INSTALL_DIR_DDB, "DynamoDBLocal.jar") @@ -195,7 +195,7 @@ def install_elasticsearch(version=None): install_dir_parent = os.path.dirname(install_dir) mkdir(install_dir_parent) # download and extract archive - tmp_archive = os.path.join(config.dirs.tmp, f"localstack.{os.path.basename(es_url)}") + tmp_archive = os.path.join(config.dirs.cache, f"localstack.{os.path.basename(es_url)}") download_and_extract_with_retry(es_url, tmp_archive, install_dir_parent) elasticsearch_dir = glob.glob(os.path.join(install_dir_parent, "elasticsearch*")) if not elasticsearch_dir: @@ -284,7 +284,7 @@ def install_opensearch(version=None): mkdir(install_dir_parent) # download and extract archive tmp_archive = os.path.join( - config.dirs.tmp, f"localstack.{os.path.basename(opensearch_url)}" + config.dirs.cache, f"localstack.{os.path.basename(opensearch_url)}" ) download_and_extract_with_retry(opensearch_url, tmp_archive, install_dir_parent) opensearch_dir = glob.glob(os.path.join(install_dir_parent, "opensearch*")) @@ -319,7 +319,7 @@ def install_elasticmq(): log_install_msg("ElasticMQ") mkdir(INSTALL_DIR_ELASTICMQ) # download archive - tmp_archive = os.path.join(config.dirs.tmp, "elasticmq-server.jar") + tmp_archive = os.path.join(config.dirs.cache, "elasticmq-server.jar") if not os.path.exists(tmp_archive): download(ELASTICMQ_JAR_URL, tmp_archive) shutil.copy(tmp_archive, INSTALL_DIR_ELASTICMQ) @@ -799,6 +799,8 @@ def install(self, package: str, *args, **kwargs): def main(): if len(sys.argv) > 1: + config.dirs.mkdirs() + # set test API key so pro install hooks are called os.environ["LOCALSTACK_API_KEY"] = os.environ.get("LOCALSTACK_API_KEY") or "test" if sys.argv[1] == "libs": diff --git a/localstack/services/kms/kms_starter.py b/localstack/services/kms/kms_starter.py index 11c07c9f27d11..a50c165b568b0 100644 --- a/localstack/services/kms/kms_starter.py +++ b/localstack/services/kms/kms_starter.py @@ -26,7 +26,7 @@ def start_kms_local(port=None, backend_port=None, asynchronous=None, update_list "KMS_ACCOUNT_ID": account_id, "ACCOUNT_ID": account_id, } - if config.dirs.data: + if config.PERSISTENCE: env_vars["KMS_DATA_PATH"] = config.dirs.data result = do_run(kms_binary, asynchronous, env_vars=env_vars) wait_for_port_open(backend_port) diff --git a/localstack/services/opensearch/cluster.py b/localstack/services/opensearch/cluster.py index 7146176528567..3c3e6a258756b 100644 --- a/localstack/services/opensearch/cluster.py +++ b/localstack/services/opensearch/cluster.py @@ -55,7 +55,7 @@ def init_directories(dirs: Directories): LOG.debug("initializing cluster directories %s", dirs) chmod_r(dirs.install, 0o777) - if not dirs.data.startswith(config.dirs.data): + if not config.dirs.data or not dirs.data.startswith(config.dirs.data): # only clear previous data if it's not in DATA_DIR rm_rf(dirs.data) diff --git a/localstack/utils/bootstrap.py b/localstack/utils/bootstrap.py index 35eeeeabe33cf..7de661807a261 100644 --- a/localstack/utils/bootstrap.py +++ b/localstack/utils/bootstrap.py @@ -12,6 +12,7 @@ from localstack import config, constants from localstack.config import Directories +from localstack.constants import DEFAULT_VOLUME_DIR from localstack.runtime import hooks from localstack.utils.container_networking import get_main_container_name from localstack.utils.container_utils.container_client import ( @@ -645,6 +646,10 @@ def configure_container(container: LocalstackContainer): def configure_volume_mounts(container: LocalstackContainer): + if not config.LEGACY_DIRECTORIES: + container.volumes.add(VolumeBind(config.VOLUME_DIR, DEFAULT_VOLUME_DIR)) + return + source_dirs = config.dirs target_dirs = Directories.for_container() @@ -663,9 +668,6 @@ def configure_volume_mounts(container: LocalstackContainer): container.volumes.add(VolumeBind(source_dirs.data, target_dirs.data)) container.env_vars["DATA_DIR"] = target_dirs.data - if source_dirs.init: - container.volumes.add(VolumeBind(source_dirs.init, target_dirs.init)) - @log_duration() def prepare_host(): diff --git a/localstack/utils/container_utils/container_client.py b/localstack/utils/container_utils/container_client.py index e7d833b79fd3f..7e4f018c55c64 100644 --- a/localstack/utils/container_utils/container_client.py +++ b/localstack/utils/container_utils/container_client.py @@ -11,12 +11,12 @@ from abc import ABCMeta, abstractmethod from enum import Enum, unique from pathlib import Path -from typing import Dict, List, Optional, Tuple, Union +from typing import Dict, List, NamedTuple, Optional, Tuple, Union if sys.version_info >= (3, 8): - from typing import Protocol, get_args + from typing import Literal, Protocol, get_args else: - from typing_extensions import Protocol, get_args + from typing_extensions import Protocol, get_args, Literal from localstack import config from localstack.utils.collections import HashableList @@ -324,6 +324,22 @@ def __iter__(self): return self.mappings.__iter__() +VolumeType = Literal["bind", "volume"] + + +class VolumeInfo(NamedTuple): + """Container volume information.""" + + type: VolumeType + source: str + destination: str + mode: str + rw: bool + propagation: str + name: Optional[str] = None + driver: Optional[str] = None + + @dataclasses.dataclass class ContainerConfiguration: image_name: str @@ -510,12 +526,24 @@ def stream_container_logs(self, container_name_or_id: str) -> CancellableStream: @abstractmethod def inspect_container(self, container_name_or_id: str) -> Dict[str, Union[Dict, str]]: - """Get detailed attributes of an container. + """Get detailed attributes of a container. :return: Dict containing docker attributes as returned by the daemon """ pass + def inspect_container_volumes(self, container_name_or_id) -> List[VolumeInfo]: + """Return information about the volumes mounted into the given container. + + :param container_name_or_id: the container name or id + :return: a list of volumes + """ + volumes = [] + for doc in self.inspect_container(container_name_or_id)["Mounts"]: + volumes.append(VolumeInfo(**{k.lower(): v for k, v in doc.items()})) + + return volumes + @abstractmethod def inspect_image(self, image_name: str, pull: bool = True) -> Dict[str, Union[Dict, str]]: """Get detailed attributes of an image. diff --git a/localstack/utils/diagnose.py b/localstack/utils/diagnose.py index 95e743e074522..600b1a6a259bc 100644 --- a/localstack/utils/diagnose.py +++ b/localstack/utils/diagnose.py @@ -5,6 +5,7 @@ from typing import Dict, List, Union from localstack import config +from localstack.constants import DEFAULT_VOLUME_DIR from localstack.utils import bootstrap from localstack.utils.container_networking import get_main_container_name from localstack.utils.container_utils.container_client import NoSuchImage @@ -68,7 +69,7 @@ "__spec__", } ENDPOINT_RESOLVE_LIST = ["localhost.localstack.cloud", "api.localstack.cloud"] -INSPECT_DIRECTORIES = ["/var/lib/localstack", "/tmp"] +INSPECT_DIRECTORIES = [DEFAULT_VOLUME_DIR, "/tmp"] def get_localstack_logs() -> Union[str, Dict]: diff --git a/localstack/utils/docker_utils.py b/localstack/utils/docker_utils.py index aea3d61cecb58..1a7446ab6bf88 100644 --- a/localstack/utils/docker_utils.py +++ b/localstack/utils/docker_utils.py @@ -1,7 +1,11 @@ +import functools import logging +import platform +from typing import List, Optional from localstack import config -from localstack.utils.container_utils.container_client import ContainerClient +from localstack.constants import DEFAULT_VOLUME_DIR +from localstack.utils.container_utils.container_client import ContainerClient, VolumeInfo """Type alias for a simple version of VolumeBind""" @@ -40,4 +44,40 @@ def create_docker_client() -> ContainerClient: return SdkDockerClient() +def get_current_container_id() -> str: + """ + Returns the ID of the current container, or raises a ValueError if we're not in docker. + + :return: the ID of the current container + """ + if not config.is_in_docker: + raise ValueError("not in docker") + + container_id = platform.node() + if not container_id: + raise OSError("no hostname returned to use as container id") + + return container_id + + +def inspect_current_container_mounts() -> List[VolumeInfo]: + return DOCKER_CLIENT.inspect_container_volumes(get_current_container_id()) + + +@functools.lru_cache() +def get_default_volume_dir_mount() -> Optional[VolumeInfo]: + """ + Returns the volume information of LocalStack's DEFAULT_VOLUME_DIR (/var/lib/localstack), if mounted, + else it returns None. If we're not currently in docker a VauleError is raised. in a container, a ValueError is + raised. + + :return: the volume info of the default volume dir or None + """ + for volume in inspect_current_container_mounts(): + if volume.destination.rstrip("/") == DEFAULT_VOLUME_DIR: + return volume + + return None + + DOCKER_CLIENT: ContainerClient = create_docker_client() diff --git a/localstack/utils/kinesis/kclipy_helper.py b/localstack/utils/kinesis/kclipy_helper.py index 237c8ba6627e1..df469c60637e1 100644 --- a/localstack/utils/kinesis/kclipy_helper.py +++ b/localstack/utils/kinesis/kclipy_helper.py @@ -5,6 +5,7 @@ from amazon_kclpy import kcl +from localstack import config from localstack.utils.aws import aws_stack from localstack.utils.files import save_file @@ -53,17 +54,7 @@ def get_kcl_classpath(properties=None, paths=None): # add path of custom java code dir_name = os.path.dirname(os.path.realpath(__file__)) paths.insert( - 0, - os.path.realpath( - os.path.join( - dir_name, - "..", - "..", - "infra", - "amazon-kinesis-client", - "aws-java-sdk-sts.jar", - ) - ), + 0, os.path.join(config.dirs.static_libs, "amazon-kinesis-client", "aws-java-sdk-sts.jar") ) paths.insert(0, os.path.realpath(os.path.join(dir_name, "java"))) return ":".join([p for p in paths if p != ""]) diff --git a/localstack/utils/persistence.py b/localstack/utils/persistence.py index 21a0a374f6dc3..efda1508e2dbe 100644 --- a/localstack/utils/persistence.py +++ b/localstack/utils/persistence.py @@ -209,7 +209,7 @@ def restore_persisted_data(apis): def is_persistence_enabled(): - return bool(config.dirs.data) + return config.PERSISTENCE and config.dirs.data def is_persistence_restored(): diff --git a/tests/bootstrap/test_cli.py b/tests/bootstrap/test_cli.py index f4fe8400a5bdf..2b42b48f327fb 100644 --- a/tests/bootstrap/test_cli.py +++ b/tests/bootstrap/test_cli.py @@ -110,6 +110,9 @@ def test_custom_docker_flags(self, runner, tmp_path, monkeypatch, container_clie assert "42069/tcp" in inspect["HostConfig"]["PortBindings"] assert f"{volume}:{volume}" in inspect["HostConfig"]["Binds"] + @pytest.mark.skipif( + condition=not config.LEGACY_DIRECTORIES, reason="this test targets LEGACY_DIRECTORIES=1" + ) def test_directories_mounted_correctly(self, runner, tmp_path, monkeypatch, container_client): data_dir = tmp_path / "data_dir" tmp_folder = tmp_path / "tmp" @@ -119,7 +122,7 @@ def test_directories_mounted_correctly(self, runner, tmp_path, monkeypatch, cont monkeypatch.setattr(config, "DATA_DIR", str(data_dir)) monkeypatch.setattr(config, "TMP_FOLDER", str(tmp_folder)) # reload directories from manipulated config - monkeypatch.setattr(config, "dirs", config.Directories.from_config()) + monkeypatch.setattr(config, "dirs", config.Directories.legacy_from_config()) runner.invoke(cli, ["start", "-d"]) runner.invoke(cli, ["wait", "-t", "60"]) @@ -131,3 +134,21 @@ def test_directories_mounted_correctly(self, runner, tmp_path, monkeypatch, cont assert f"{tmp_folder}:{container_dirs.tmp}" in binds assert f"{data_dir}:{container_dirs.data}" in binds assert f"{DOCKER_SOCK}:{DOCKER_SOCK}" in binds + + @pytest.mark.skipif( + condition=config.LEGACY_DIRECTORIES, reason="this test targets LEGACY_DIRECTORIES=0" + ) + def test_volume_dir_mounted_correctly(self, runner, tmp_path, monkeypatch, container_client): + volume_dir = tmp_path / "volume" + + # set different directories and make sure they are mounted correctly + monkeypatch.setenv("LOCALSTACK_VOLUME_DIR", str(volume_dir)) + monkeypatch.setattr(config, "VOLUME_DIR", str(volume_dir)) + + runner.invoke(cli, ["start", "-d"]) + runner.invoke(cli, ["wait", "-t", "60"]) + + # check that mounts were created correctly + inspect = container_client.inspect_container(config.MAIN_CONTAINER_NAME) + binds = inspect["HostConfig"]["Binds"] + assert f"{volume_dir}:{constants.DEFAULT_VOLUME_DIR}" in binds diff --git a/tests/integration/awslambda/test_lambda_whitebox.py b/tests/integration/awslambda/test_lambda_whitebox.py index 5b5f17cd3796b..d7e9a904da8c0 100644 --- a/tests/integration/awslambda/test_lambda_whitebox.py +++ b/tests/integration/awslambda/test_lambda_whitebox.py @@ -219,7 +219,7 @@ def test_code_updated_on_redeployment(self): ) def test_prime_and_destroy_containers(self): executor = lambda_api.LAMBDA_EXECUTOR - func_name = "test_prime_and_destroy_containers" + func_name = f"test_prime_and_destroy_containers_{short_uid()}" func_arn = lambda_api.func_arn(func_name) # make sure existing containers are gone diff --git a/tests/integration/docker_utils/test_docker.py b/tests/integration/docker_utils/test_docker.py index 0fe56a192278d..aa4bc653936b6 100644 --- a/tests/integration/docker_utils/test_docker.py +++ b/tests/integration/docker_utils/test_docker.py @@ -22,7 +22,9 @@ PortMappings, RegistryConnectionError, Util, + VolumeInfo, ) +from localstack.utils.container_utils.docker_cmd_client import CmdDockerClient from localstack.utils.net import get_free_tcp_port ContainerInfo = NamedTuple( @@ -386,6 +388,79 @@ def test_create_with_volume(self, tmpdir, docker_client: ContainerClient, create docker_client.start_container(c.container_id) assert tmpdir.join("foo.log").isfile(), "foo.log was not created in mounted dir" + @pytest.mark.skipif( + condition=in_docker(), reason="cannot test volume mounts from host when in docker" + ) + def test_inspect_container_volumes( + self, tmpdir, docker_client: ContainerClient, create_container + ): + mount_volumes = [ + (tmpdir.realpath() / "foo", "/tmp/mypath/foo"), + ("some_named_volume", "/tmp/mypath/volume"), + ] + + c = create_container( + "alpine", + command=["sh", "-c", "while true; do sleep 1; done"], + mount_volumes=mount_volumes, + ) + docker_client.start_container(c.container_id) + + vols = docker_client.inspect_container_volumes(c.container_id) + + # FIXME cmd docker client creates different default permission mode flags + if isinstance(docker_client, CmdDockerClient): + vol1 = VolumeInfo( + type="bind", + source=f"{tmpdir}/foo", + destination="/tmp/mypath/foo", + mode="", + rw=True, + propagation="rprivate", + name=None, + driver=None, + ) + vol2 = VolumeInfo( + type="volume", + source="/var/lib/docker/volumes/some_named_volume/_data", + destination="/tmp/mypath/volume", + mode="z", + rw=True, + propagation="", + name="some_named_volume", + driver="local", + ) + else: + vol1 = VolumeInfo( + type="bind", + source=f"{tmpdir}/foo", + destination="/tmp/mypath/foo", + mode="rw", + rw=True, + propagation="rprivate", + name=None, + driver=None, + ) + vol2 = VolumeInfo( + type="volume", + source="/var/lib/docker/volumes/some_named_volume/_data", + destination="/tmp/mypath/volume", + mode="rw", + rw=True, + propagation="", + name="some_named_volume", + driver="local", + ) + + assert vol1 in vols + assert vol2 in vols + + def test_inspect_container_volumes_with_no_volumes( + self, docker_client: ContainerClient, dummy_container + ): + docker_client.start_container(dummy_container.container_id) + assert len(docker_client.inspect_container_volumes(dummy_container.container_id)) == 0 + def test_copy_into_container(self, tmpdir, docker_client: ContainerClient, create_container): local_path = tmpdir.join("myfile.txt") container_path = "/tmp/myfile_differentpath.txt" diff --git a/tests/unit/test_lambda.py b/tests/unit/test_lambda.py index 9be16db42343f..0b14ef9e942c3 100644 --- a/tests/unit/test_lambda.py +++ b/tests/unit/test_lambda.py @@ -7,8 +7,8 @@ import mock +from localstack import config from localstack.aws.accounts import get_aws_account_id -from localstack.config import TMP_FOLDER from localstack.services.awslambda import lambda_api, lambda_executors, lambda_utils from localstack.services.awslambda.lambda_executors import OutputLog from localstack.services.awslambda.lambda_utils import API_PATH_ROOT @@ -1067,7 +1067,9 @@ def test_lambda_output(self, temp): output.output_file() - temp.assert_called_once_with(dir=TMP_FOLDER, delete=False, suffix=".log", prefix="lambda_") + temp.assert_called_once_with( + dir=config.dirs.tmp, delete=False, suffix=".log", prefix="lambda_" + ) class TestLambdaEventInvokeConfig(unittest.TestCase):