odtp-org · sabinem · Jun 19, 2024 · May 13, 2024 · May 13, 2024 · May 15, 2024
diff --git a/.env.dist.compose b/.env.dist.compose
@@ -32,4 +32,14 @@ MONGODB_PATH=
 
 # Dashboard parameters
 ODTP_DASHBOARD_PORT=
-ODTP_DASHBOARD_RELOAD=
+ODTP_DASHBOARD_JSON_EDITOR=
+
+# Log level
+ODTP_LOG_LEVEL=
+RUN_LOG_LEVEL=
+
+# App Path
+APP_PATH=
+
+# Install the package in editable mode.
+PIP_INSTALL_ARGS="--editable"
diff --git a/.env.dist.local b/.env.dist.local
@@ -1,7 +1,7 @@
 # environment variables for local installation
 # ----------------------------------------------
-# fill these variables in case you want to 
-# install otdp locally on your computer or 
+# fill these variables in case you want to
+# install otdp locally on your computer or
 # on a server
 
 # mongo url: example "mongodb://localhost:27017/"
@@ -12,19 +12,23 @@ ODTP_S3_SERVER=
 # odtp db instance in the mongo db: "odtp"
 ODTP_MONGO_DB=
 
-# s3 bucket name: "odtp" 
+# s3 bucket name: "odtp"
 ODTP_BUCKET_NAME=
 
 # s3 access and secret key
-ODTP_ACCESS_KEY=     
+ODTP_ACCESS_KEY=
 ODTP_SECRET_KEY=
 
 # your github token
 GITHUB_TOKEN=
 
 # Dashboard parameters
 ODTP_DASHBOARD_PORT=
-ODTP_DASHBOARD_RELOAD=
+ODTP_DASHBOARD_JSON_EDITOR=
 
 # Working directory for user projects
 ODTP_PATH=
+
+# Log level
+ODTP_LOG_LEVEL=
+RUN_LOG_LEVEL=
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,6 @@
 __pycache__/
 .DS_Store
-.env
+.env*
+!.env*.dist
 .nicegui
-.env.bk
+.local
diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,6 @@
 # Use Docker in Docker stable version
 FROM docker:25.0.3-dind-alpine3.19
+ARG PIP_INSTALL_ARGS=
 
 # Install dependencies required
 RUN apk add --no-cache \
@@ -11,7 +12,7 @@ RUN apk add --no-cache \
     zlib-dev \
     git \
     curl \
-    wget 
+    wget
 
 # Install Rust and Cargo using rustup (the Rust toolchain installer)
 # Note: The following command installs the stable version of Rust and adds Cargo to the PATH
@@ -20,7 +21,7 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
 # Install system dependencies required for compiling Fortran and C++ code
 # Needed for Numpy.
 RUN apk add --no-cache \
-    curl \ 
+    curl \
     gfortran \
     g++ \
     cmake \
@@ -60,17 +61,12 @@ RUN pip3 install --upgrade pip && \
 
 # Add your application's source code
 COPY . /app
-
 WORKDIR /app
-
-RUN pip3 install --upgrade pip && \
-    pip3 install --break-system-packages .
+RUN pip3 install --break-system-packages $PIP_INSTALL_ARGS .
 
 # Symbolic link between python and python3
 RUN rm /usr/bin/python
 RUN ln -s /usr/local/bin/python3 /usr/bin/python
 
-WORKDIR /app
-
 # Entry point in a sh session
 ENTRYPOINT ["sh"]
diff --git a/compose.dev.yml b/compose.dev.yml
@@ -0,0 +1,90 @@
+version: "3.8"
+
+services:
+  odtp:
+    build:
+      context: .
+      args:
+        PIP_INSTALL_ARGS: ${PIP_INSTALL_ARGS}
+    image: odtp
+    privileged: true
+    ports:
+      - ${ODTP_DASHBOARD_PORT}:${ODTP_DASHBOARD_PORT}
+      - "8501:8501"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ${ODTP_PATH}:${ODTP_PATH}
+      - ${APP_PATH}:/app
+    environment:
+      ODTP_PATH: ${ODTP_PATH}
+      ODTP_MONGO_SERVER: mongodb://${MONGO_DB_USER}:${MONGO_DB_PASSWORD}@mongodb-instance:27017/
+      ODTP_S3_SERVER: http://minio-instance:9000
+      ODTP_BUCKET_NAME: ${ODTP_BUCKET_NAME}
+      ODTP_ACCESS_KEY: ${ODTP_ACCESS_KEY}
+      ODTP_SECRET_KEY: ${ODTP_SECRET_KEY}
+      GITHUB_TOKEN: ${GITHUB_TOKEN}
+      ODTP_MONGO_DB: ${ODTP_MONGO_DB}
+      ODTP_DASHBOARD_PORT: ${ODTP_DASHBOARD_PORT}
+      ODTP_DASHBOARD_RELOAD: ${ODTP_DASHBOARD_RELOAD}
+      ODTP_DASHBOARD_JSON_EDITIOR: ${ODTP_DASHBOARD_JSON_EDITIOR}
+      ODTP_LOG_LEVEL: ${ODTP_LOG_LEVEL}
+      RUN_LOG_LEVEL: ${ODTP_LOG_LEVEL}
+
+    stdin_open: true
+    tty: true
+    networks:
+      - odtp-network
+    depends_on:
+      - mongodb-instance
+      - mongodb-express
+      - minio-instance
+
+  mongodb-instance:
+    image: mongo:7.0.8
+    ports:
+      - "27017:27017"
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: ${MONGO_DB_USER}
+      MONGO_INITDB_ROOT_PASSWORD: ${MONGO_DB_PASSWORD}
+      MONGO_INITDB_DATABASE: ${ODTP_MONGO_DB}
+    networks:
+      - odtp-network
+    volumes:
+      - ${MONGODB_PATH}:/data/db
+
+  mongodb-express:
+    image: mongo-express:1.0.2-18
+    ports:
+      - "8081:8081"
+    environment:
+      ME_CONFIG_OPTIONS_EDITORTHEME: "ambiance"
+      ME_CONFIG_BASICAUTH_USERNAME: ${MONGO_EXPRESS_USER}
+      ME_CONFIG_BASICAUTH_PASSWORD: ${MONGO_EXPRESS_PASSWORD}
+      ME_CONFIG_MONGODB_URL: mongodb://${MONGO_DB_USER}:${MONGO_DB_PASSWORD}@mongodb-instance:27017/
+      ME_CONFIG_MONGODB_ENABLE_ADMIN: true
+    networks:
+      - odtp-network
+    depends_on:
+      - mongodb-instance
+
+  minio-instance:
+    image: minio/minio:RELEASE.2024-03-30T09-41-56Z
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    environment:
+      MINIO_ROOT_USER: ${ODTP_ACCESS_KEY}
+      MINIO_ROOT_PASSWORD: ${ODTP_SECRET_KEY}
+    command: server --console-address :9001 /data
+    networks:
+      - odtp-network
+    volumes:
+      - ${MINIO_PATH}:/data
+
+volumes:
+  mongodb_data:
+  minio_data:
+
+networks:
+  odtp-network:
+    driver: bridge
diff --git a/compose.yml → compose.prod.yml b/compose.yml → compose.prod.yml
@@ -1,4 +1,4 @@
-version: '3.8'
+version: "3.8"
 
 services:
   odtp:
@@ -11,6 +11,7 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       - ${ODTP_PATH}:${ODTP_PATH}
+      - ${APP_PATH}:/app
     environment:
       ODTP_PATH: ${ODTP_PATH}
       ODTP_MONGO_SERVER: mongodb://${MONGO_DB_USER}:${MONGO_DB_PASSWORD}@mongodb-instance:27017/
@@ -21,8 +22,10 @@ services:
       GITHUB_TOKEN: ${GITHUB_TOKEN}
       ODTP_MONGO_DB: ${ODTP_MONGO_DB}
       ODTP_DASHBOARD_PORT: ${ODTP_DASHBOARD_PORT}
-      ODTP_DASHBOARD_RELOAD: ${ODTP_DASHBOARD_RELOAD}
-
+      ODTP_DASHBOARD_JSON_EDITIOR: ${ODTP_DASHBOARD_JSON_EDITIOR}
+      ODTP_LOG_LEVEL: ${ODTP_LOG_LEVEL}
+      RUN_LOG_LEVEL: ${ODTP_LOG_LEVEL}
+
     stdin_open: true
     tty: true
     networks:
@@ -39,7 +42,7 @@ services:
     environment:
       MONGO_INITDB_ROOT_USERNAME: ${MONGO_DB_USER}
       MONGO_INITDB_ROOT_PASSWORD: ${MONGO_DB_PASSWORD}
-      MONGO_INITDB_DATABASE: ${ODTP_MONGO_DB}   
+      MONGO_INITDB_DATABASE: ${ODTP_MONGO_DB}
     networks:
       - odtp-network
     volumes:

diff --git a/odtp/__init__.py b/odtp/__init__.py
@@ -5,12 +5,18 @@
 
 import logging
 import importlib.metadata as importlib_metadata
+from odtp.helpers.settings import ODTP_LOG_LEVEL
+
 
 __version__ = importlib_metadata.version(__name__)
-logging.basicConfig(#filename='my_logs.log',
-                    #encoding='utf-8',
-                    #filemode='w',
-                    format='%(levelname)s (%(asctime)s): %(message)s (LineL %(lineno)d [%(filename)s])',
-                    datefmt='%d/%m/%Y %I:%M:%S %p',
-                    level=logging.INFO)
-# logging.info('The answer is: %s', x) #Variable.
+
+logging.basicConfig(
+    format='%(asctime)s - [%(module)s:%(levelname)s] %(lineno)d %(filename)s %(funcName)s - %(message)s',
+    datefmt='%d/%m/%Y %I:%M:%S %p',
+    level=logging.ERROR
+)
+
+valid_log_levels = logging.getLevelNamesMapping()
+
+if ODTP_LOG_LEVEL in valid_log_levels.keys():
+    logging.getLogger().setLevel(ODTP_LOG_LEVEL)
diff --git a/odtp/barfi.py b/odtp/barfi.py
diff --git a/odtp/cli/component.py b/odtp/cli/component.py
@@ -60,7 +60,7 @@ def run(
         help="Specify the path to the environment file"
     )] = None,     
     ports: Annotated[str, typer.Option(
-        help="Specify port mappings seperated by a plus sign i.e. 8501:8501+8201:8201"
+        help="Specify port mappings separated by a plus sign i.e. 8501:8501+8201:8201"
     )] = None,  
 ):
     try:
@@ -71,7 +71,7 @@ def run(
             image_name=image_name, 
         )
         ports = odtp_parse.parse_port_mappings_for_one_component(ports) 
-        parameters = odtp_parse.parse_paramters_for_one_file(parameter_file)
+        parameters = odtp_parse.parse_parameters_for_one_file(parameter_file)
         componentManager.run_component(
             parameters=parameters, 
             ports=ports, 

diff --git a/odtp/cli/execution.py b/odtp/cli/execution.py
@@ -1,7 +1,6 @@
 """
 This scripts contains odtp subcommands for 'execution'
 """
-import logging
 
 import typer
 from typing_extensions import Annotated
@@ -104,7 +103,7 @@ def output(
         display_tree(project_path)
     except Exception as e:
         print(f"ERROR: Output printing failed: {e}")       
-        raise typer.Abort()  
+        raise typer.Abort()
 
 
 if __name__ == "__main__":

diff --git a/odtp/cli/new.py b/odtp/cli/new.py
@@ -7,7 +7,6 @@
 import odtp.mongodb.db as db
 import odtp.helpers.parse as odtp_parse
 import odtp.mongodb.utils as db_utils
-import odtp.helpers.utils as odtp_utils
 import odtp.helpers.git as odtp_git