localstack · thrau · Jun 28, 2022 · Jun 18, 2022 · Jun 19, 2022 · Jun 19, 2022
diff --git a/.gitignore b/.gitignore
@@ -6,6 +6,8 @@
 .coverage.*
 htmlcov
 
+.cache
+.filesystem
 /infra/
 localstack/infra/
 

diff --git a/Dockerfile b/Dockerfile
@@ -90,6 +90,16 @@ ADD https://raw.githubusercontent.com/carlossg/docker-maven/master/openjdk-11/se
 RUN mkdir -p /opt/code/localstack
 WORKDIR /opt/code/localstack/
 
+# create filesystem hierarchy
+RUN mkdir -p /var/lib/localstack && \
+    mkdir -p /usr/lib/localstack
+# backwards compatibility with LEGACY_DIRECTORIES (TODO: deprecate and remove)
+RUN mkdir -p /opt/code/localstack/localstack && \
+    ln -s /usr/lib/localstack /opt/code/localstack/localstack/infra && \
+    mkdir /tmp/localstack && \
+    chmod -R 777 /tmp/localstack && \
+    chmod -R 777 /usr/lib/localstack
+
 # install basic (global) tools to final image
 RUN pip install --no-cache-dir --upgrade supervisor virtualenv
 
@@ -130,12 +140,9 @@ RUN (cd /tmp && git clone https://github.com/timescale/timescaledb.git) && \
 
 # init environment and cache some dependencies
 ARG DYNAMODB_ZIP_URL=https://s3-us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_latest.zip
-RUN mkdir -p /opt/code/localstack/localstack/infra/dynamodb && \
+RUN mkdir -p /usr/lib/localstack/dynamodb && \
       curl -L -o /tmp/localstack.ddb.zip ${DYNAMODB_ZIP_URL} && \
-      (cd localstack/infra/dynamodb && unzip -q /tmp/localstack.ddb.zip && rm /tmp/localstack.ddb.zip) && \
-    mkdir -p /opt/code/localstack/localstack/infra/elasticmq && \
-    curl -L -o /opt/code/localstack/localstack/infra/elasticmq/elasticmq-server.jar \
-        https://s3-eu-west-1.amazonaws.com/softwaremill-public/elasticmq-server-1.1.0.jar
+      (cd /usr/lib/localstack/dynamodb && unzip -q /tmp/localstack.ddb.zip && rm /tmp/localstack.ddb.zip)
 
 # upgrade python build tools
 RUN (virtualenv .venv && source .venv/bin/activate && pip3 install --upgrade pip wheel setuptools)
@@ -158,8 +165,7 @@ RUN (virtualenv .venv && source .venv/bin/activate && pip3 uninstall -y localsta
 
 # base-light: Stage which does not add additional dependencies (like elasticsearch)
 FROM base as base-light
-RUN mkdir -p /opt/code/localstack/localstack/infra && \
-    touch localstack/infra/.light-version
+RUN touch /usr/lib/localstack/.light-version
 
 
 
@@ -170,13 +176,12 @@ FROM base as base-full
 # https://github.com/pires/docker-elasticsearch/issues/56
 ENV ES_TMPDIR /tmp
 
-ENV ES_BASE_DIR=localstack/infra/elasticsearch
+ENV ES_BASE_DIR=/usr/lib/localstack/elasticsearch
 ENV ES_JAVA_HOME /usr/lib/jvm/java-11
 RUN TARGETARCH_SYNONYM=$([[ "$TARGETARCH" == "amd64" ]] && echo "x86_64" || echo "aarch64"); \
-    mkdir -p /opt/code/localstack/localstack/infra && \
     curl -L -o /tmp/localstack.es.tar.gz \
         https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.10.0-linux-${TARGETARCH_SYNONYM}.tar.gz && \
-    (cd localstack/infra/ && tar -xf /tmp/localstack.es.tar.gz && \
+    (cd /usr/lib/localstack && tar -xf /tmp/localstack.es.tar.gz && \
         mv elasticsearch* elasticsearch && rm /tmp/localstack.es.tar.gz) && \
     (cd $ES_BASE_DIR && \
         bin/elasticsearch-plugin install analysis-icu && \
@@ -213,24 +218,24 @@ COPY --from=builder /opt/code/localstack/ /opt/code/localstack/
 COPY --from=builder /usr/share/postgresql/11/extension /usr/share/postgresql/11/extension
 COPY --from=builder /usr/lib/postgresql/11/lib /usr/lib/postgresql/11/lib
 
-RUN mkdir -p /tmp/localstack && \
-    if [ -e /usr/bin/aws ]; then mv /usr/bin/aws /usr/bin/aws.bk; fi; ln -s /opt/code/localstack/.venv/bin/aws /usr/bin/aws
+RUN if [ -e /usr/bin/aws ]; then mv /usr/bin/aws /usr/bin/aws.bk; fi; ln -s /opt/code/localstack/.venv/bin/aws /usr/bin/aws
 
 # fix some permissions and create local user
 RUN mkdir -p /.npm && \
     chmod 777 . && \
     chmod 755 /root && \
     chmod -R 777 /.npm && \
-    chmod -R 777 /tmp/localstack && \
+    chmod -R 777 /var/lib/localstack && \
     useradd -ms /bin/bash localstack && \
     ln -s `pwd` /tmp/localstack_install_dir
 
 # Install the latest version of awslocal globally
 RUN pip3 install --upgrade awscli awscli-local requests
 
-# Add the code in the last step
-# Also adds the results of `make init` to the container.
+# Adds the results of `make init` to the container.
 # `make init` _needs_ to be executed before building this docker image (since the execution needs docker itself).
+ADD .filesystem/usr/lib/localstack /usr/lib/localstack
+# Add the code in the last step
 ADD localstack/ localstack/
 
 # Download some more dependencies (make init needs the LocalStack code)
@@ -259,5 +264,8 @@ EXPOSE 4566 4510-4559 5678
 
 HEALTHCHECK --interval=10s --start-period=15s --retries=5 --timeout=5s CMD ./bin/localstack status services --format=json
 
+# default volume directory
+VOLUME /var/lib/localstack
+
 # define command at startup
 ENTRYPOINT ["docker-entrypoint.sh"]
diff --git a/Makefile b/Makefile
@@ -193,7 +193,7 @@ docker-run:        		  ## Run Docker image locally
 
 docker-mount-run:
 	MOTO_DIR=$$(echo $$(pwd)/.venv/lib/python*/site-packages/moto | awk '{print $$NF}'); echo MOTO_DIR $$MOTO_DIR; \
-		DOCKER_FLAGS="$(DOCKER_FLAGS) -v `pwd`/localstack/constants.py:/opt/code/localstack/localstack/constants.py -v `pwd`/localstack/config.py:/opt/code/localstack/localstack/config.py -v `pwd`/localstack/plugins.py:/opt/code/localstack/localstack/plugins.py -v `pwd`/localstack/plugin:/opt/code/localstack/localstack/plugin -v `pwd`/localstack/runtime:/opt/code/localstack/localstack/runtime -v `pwd`/localstack/utils:/opt/code/localstack/localstack/utils -v `pwd`/localstack/services:/opt/code/localstack/localstack/services -v `pwd`/localstack/http:/opt/code/localstack/localstack/http -v `pwd`/localstack/contrib:/opt/code/localstack/localstack/contrib -v `pwd`/localstack/dashboard:/opt/code/localstack/localstack/dashboard -v `pwd`/tests:/opt/code/localstack/tests -v $$MOTO_DIR:/opt/code/localstack/.venv/lib/python3.8/site-packages/moto/" make docker-run
+		DOCKER_FLAGS="$(DOCKER_FLAGS) -v `pwd`/localstack/constants.py:/opt/code/localstack/localstack/constants.py -v `pwd`/localstack/config.py:/opt/code/localstack/localstack/config.py -v `pwd`/localstack/plugins.py:/opt/code/localstack/localstack/plugins.py -v `pwd`/localstack/plugin:/opt/code/localstack/localstack/plugin -v `pwd`/localstack/runtime:/opt/code/localstack/localstack/runtime -v `pwd`/localstack/utils:/opt/code/localstack/localstack/utils -v `pwd`/localstack/services:/opt/code/localstack/localstack/services -v `pwd`/localstack/http:/opt/code/localstack/localstack/http -v `pwd`/localstack/contrib:/opt/code/localstack/localstack/contrib -v `pwd`/localstack/dashboard:/opt/code/localstack/localstack/dashboard -v `pwd`/tests:/opt/code/localstack/tests -v $$MOTO_DIR:/opt/code/localstack/.venv/lib/python3.10/site-packages/moto/" make docker-run
 
 docker-build-lambdas:
 	docker build -t localstack/lambda-js:nodejs14.x -f bin/lambda/Dockerfile.nodejs14x .
@@ -220,13 +220,13 @@ test-docker-mount:		  ## Run automated tests in Docker (mounting local code)
 	# TODO: find a cleaner way to mount/copy the dependencies into the container...
 	VENV_DIR=$$(pwd)/.venv/; \
 		PKG_DIR=$$(echo $$VENV_DIR/lib/python*/site-packages | awk '{print $$NF}'); \
-		PKG_DIR_CON=/opt/code/localstack/.venv/lib/python3.8/site-packages; \
+		PKG_DIR_CON=/opt/code/localstack/.venv/lib/python3.10/site-packages; \
 		echo "#!/usr/bin/env python" > /tmp/pytest.ls.bin; cat $$VENV_DIR/bin/pytest >> /tmp/pytest.ls.bin; chmod +x /tmp/pytest.ls.bin; \
-		DOCKER_FLAGS="-v `pwd`/tests:/opt/code/localstack/tests -v /tmp/pytest.ls.bin:/opt/code/localstack/.venv/bin/pytest -v $$PKG_DIR/py:$$PKG_DIR_CON/py -v $$PKG_DIR/pluggy:$$PKG_DIR_CON/pluggy -v $$PKG_DIR/iniconfig:$$PKG_DIR_CON/iniconfig -v $$PKG_DIR/packaging:$$PKG_DIR_CON/packaging -v $$PKG_DIR/pytest:$$PKG_DIR_CON/pytest -v $$PKG_DIR/_pytest:/opt/code/localstack/.venv/lib/python3.8/site-packages/_pytest" make test-docker-mount-code
+		DOCKER_FLAGS="-v `pwd`/tests:/opt/code/localstack/tests -v /tmp/pytest.ls.bin:/opt/code/localstack/.venv/bin/pytest -v $$PKG_DIR/py:$$PKG_DIR_CON/py -v $$PKG_DIR/pluggy:$$PKG_DIR_CON/pluggy -v $$PKG_DIR/iniconfig:$$PKG_DIR_CON/iniconfig -v $$PKG_DIR/packaging:$$PKG_DIR_CON/packaging -v $$PKG_DIR/pytest:$$PKG_DIR_CON/pytest -v $$PKG_DIR/_pytest:/opt/code/localstack/.venv/lib/python3.10/site-packages/_pytest" make test-docker-mount-code
 
 test-docker-mount-code:
 	PACKAGES_DIR=$$(echo $$(pwd)/.venv/lib/python*/site-packages | awk '{print $$NF}'); \
-		DOCKER_FLAGS="$(DOCKER_FLAGS) --entrypoint= -v `pwd`/localstack/config.py:/opt/code/localstack/localstack/config.py -v `pwd`/localstack/constants.py:/opt/code/localstack/localstack/constants.py -v `pwd`/localstack/utils:/opt/code/localstack/localstack/utils -v `pwd`/localstack/services:/opt/code/localstack/localstack/services -v `pwd`/localstack/aws:/opt/code/localstack/localstack/aws -v `pwd`/Makefile:/opt/code/localstack/Makefile -v $$PACKAGES_DIR/moto:/opt/code/localstack/.venv/lib/python3.8/site-packages/moto/ -e TEST_PATH=\\'$(TEST_PATH)\\' -e LAMBDA_JAVA_OPTS=$(LAMBDA_JAVA_OPTS) $(ENTRYPOINT)" CMD="make test" make docker-run
+		DOCKER_FLAGS="$(DOCKER_FLAGS) --entrypoint= -v `pwd`/localstack/config.py:/opt/code/localstack/localstack/config.py -v `pwd`/localstack/constants.py:/opt/code/localstack/localstack/constants.py -v `pwd`/localstack/utils:/opt/code/localstack/localstack/utils -v `pwd`/localstack/services:/opt/code/localstack/localstack/services -v `pwd`/localstack/aws:/opt/code/localstack/localstack/aws -v `pwd`/Makefile:/opt/code/localstack/Makefile -v $$PACKAGES_DIR/moto:/opt/code/localstack/.venv/lib/python3.10/site-packages/moto/ -e TEST_PATH=\\'$(TEST_PATH)\\' -e LAMBDA_JAVA_OPTS=$(LAMBDA_JAVA_OPTS) $(ENTRYPOINT)" CMD="make test" make docker-run
 
 # Note: the ci-* targets below should only be used in CI builds!
 
@@ -277,7 +277,9 @@ init-precommit:    		  ## install te pre-commit hook into your local git reposit
 	($(VENV_RUN); pre-commit install)
 
 clean:             		  ## Clean up (npm dependencies, downloaded infrastructure code, compiled Java classes)
+	rm -rf .filesystem
 	rm -rf localstack/dashboard/web/node_modules
+	# TODO: remove localstack/infra/ as it's no longer used
 	rm -rf localstack/infra/amazon-kinesis-client
 	rm -rf localstack/infra/elasticsearch
 	rm -rf localstack/infra/elasticmq

diff --git a/bin/docker-entrypoint.sh b/bin/docker-entrypoint.sh
@@ -5,7 +5,7 @@ shopt -s nullglob
 
 if [[ ! $INIT_SCRIPTS_PATH ]]
 then
-  # FIXME: move
+  # FIXME: move to /etc/localstack/init/ready.d
   INIT_SCRIPTS_PATH=/docker-entrypoint-initaws.d
 fi
 if [[ ! $EDGE_PORT ]]
@@ -45,14 +45,21 @@ if [ "$DISABLE_TERM_HANDLER" == "" ]; then
   trap 'kill -31 ${!}; term_handler 31' SIGUSR2
 fi
 
-cat /dev/null > /tmp/localstack_infra.log
-cat /dev/null > /tmp/localstack_infra.err
+LOG_DIR=/var/lib/localstack/logs
+test -d ${LOG_DIR} || mkdir -p ${LOG_DIR}
+
+cat /dev/null > ${LOG_DIR}/localstack_infra.log
+cat /dev/null > ${LOG_DIR}/localstack_infra.err
+
+# for backwards compatibility with LEGACY_DIRECTORIES=1
+test -f /tmp/localstack_infra.log || ln -s ${LOG_DIR}/localstack_infra.log /tmp/localstack_infra.log
+test -f /tmp/localstack_infra.err || ln -s ${LOG_DIR}/localstack_infra.err /tmp/localstack_infra.err
 
 supervisord -c /etc/supervisord.conf &
 suppid="$!"
 
 function run_startup_scripts {
-  until grep -q '^Ready.' /tmp/localstack_infra.log >/dev/null 2>&1 ; do
+  until grep -q '^Ready.' ${LOG_DIR}/localstack_infra.log >/dev/null 2>&1 ; do
     echo "Waiting for all LocalStack services to be ready"
     sleep 7
   done
@@ -73,8 +80,8 @@ run_startup_scripts &
 # Run tail on the localstack log files forever until we are told to terminate
 if [ "$DISABLE_TERM_HANDLER" == "" ]; then
   while true; do
-    tail -qF /tmp/localstack_infra.log /tmp/localstack_infra.err & wait ${!}
+    tail -qF ${LOG_DIR}/localstack_infra.log ${LOG_DIR}/localstack_infra.err & wait ${!}
   done
 else
-  tail -qF /tmp/localstack_infra.log /tmp/localstack_infra.err
+  tail -qF ${LOG_DIR}/localstack_infra.log ${LOG_DIR}/localstack_infra.err
 fi
diff --git a/bin/supervisord.conf b/bin/supervisord.conf
@@ -9,5 +9,5 @@ environment=
     LOCALSTACK_INFRA_PROCESS=1
 autostart=true
 autorestart=true
-stdout_logfile=/tmp/localstack_infra.log
-stderr_logfile=/tmp/localstack_infra.err
+stdout_logfile=/var/lib/localstack/logs/localstack_infra.log
+stderr_logfile=/var/lib/localstack/logs/localstack_infra.err
diff --git a/localstack/cli/lpm.py b/localstack/cli/lpm.py
@@ -5,6 +5,7 @@
 from click import ClickException
 from rich.console import Console
 
+from localstack import config
 from localstack.services.install import InstallerManager
 from localstack.utils.bootstrap import setup_logging
 
@@ -59,6 +60,7 @@ def install(package, parallel):
     """
     console.print(f"resolving packages: {package}")
     installers: Dict[str, Callable] = InstallerManager().get_installers()
+    config.dirs.mkdirs()
 
     for pkg in package:
         if pkg not in installers:
-Original file line number
+Diff line change
@@ Expand Up / @@ -6,6 +6,8 @@ @@
     .coverage.*
     htmlcov
+    .cache
+    .filesystem
     /infra/
     localstack/infra/
@@ Expand Down @@