dagu-org · yottahmd · Jul 18, 2024 · Jul 16, 2024 · Jul 17, 2024 · Jul 18, 2024
diff --git a/.gitignore b/.gitignore
@@ -19,7 +19,5 @@ tmp/*
 # Goland
 .idea
 
-# local development
-!local/cert
-local/cert/*
-!local/cert/openssl.conf
+# Directory for local development
+local/
diff --git a/Dockerfile b/Dockerfile
@@ -45,10 +45,10 @@ WORKDIR /home/${USER}
 
 COPY --from=go-builder /app/bin/dagu /usr/local/bin/
 
-RUN mkdir -p .dagu/dags
+RUN mkdir -p .config/dagu/dags
 
 # Add the hello_world.yaml file
-COPY <<EOF .dagu/dags/hello_world.yaml
+COPY <<EOF .config/dagu/dags/hello_world.yaml
 schedule: "* * * * *"
 steps:
   - name: hello world

diff --git a/Makefile b/Makefile
@@ -1,99 +1,130 @@
-.PHONY: build server scheduler test proto certs swagger https
+.PHONY: run run-server run-server-https run-scheduler test lint build certs swagger
 
-########## Arguments ##########
+##############################################################################
+# Arguments
+##############################################################################
 
 VERSION=
 
-########## Variables ##########
+##############################################################################
+# Variables
+##############################################################################
 
 # This Makefile's directory
 SCRIPT_DIR=$(abspath $(dir $(lastword $(MAKEFILE_LIST))))
+
+# Directories for miscellaneous files for the local environment
 LOCAL_DIR=$(SCRIPT_DIR)/local
+LOCAL_BIN_DIR=$(LOCAL_DIR)/bin
 
-SRC_DIR=$(SCRIPT_DIR)
-DST_DIR=$(SRC_DIR)/internal
+# Configuration directory
+CONFIG_DIR=$(SCRIPT_DIR)/config
 
+# Local build settings
+BIN_DIR=$(SCRIPT_DIR)/bin
 BUILD_VERSION=$(shell date +'%y%m%d%H%M%S')
 LDFLAGS=-X 'main.version=$(BUILD_VERSION)'
 
-DOCKER_CMD := docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7,linux/arm64/v8 --builder container --build-arg VERSION=$(VERSION) --push --no-cache
+# Application name
 
-DEV_CERT_SUBJ_CA="/C=TR/ST=ASIA/L=TOKYO/O=DEV/OU=DAGU/CN=*.dagu.dev/[email protected]"
-DEV_CERT_SUBJ_SERVER="/C=TR/ST=ASIA/L=TOKYO/O=DEV/OU=SERVER/CN=*.server.dev/[email protected]"
-DEV_CERT_SUBJ_CLIENT="/C=TR/ST=ASIA/L=TOKYO/O=DEV/OU=CLIENT/CN=*.client.dev/[email protected]"
-DEV_CERT_SUBJ_ALT="subjectAltName=DNS:localhost"
+APP_NAME=dagu
 
-PKG_SWAGGER=github.com/go-swagger/go-swagger/cmd/swagger
-PKG_GOLANGCI_LINT=github.com/golangci/golangci-lint/cmd/golangci-lint
-PKG_gotestsum=gotest.tools/gotestsum
+# Docker image build configuration
+DOCKER_CMD := docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7,linux/arm64/v8 --builder container --build-arg VERSION=$(VERSION) --push --no-cache
 
-COLOR_GREEN=\033[0;32m
-COLOR_RESET=\033[0m
+# Arguments for the tests
+GOTESTSUM_ARGS=--format=standard-quiet
+GO_TEST_FLAGS=-v --race
+
+# Frontend directories
 
 FE_DIR=./internal/frontend
 FE_GEN_DIR=${FE_DIR}/gen
 FE_ASSETS_DIR=${FE_DIR}/assets
+FE_BUILD_DIR=./ui/dist
+FE_BUNDLE_JS=${FE_ASSETS_DIR}/bundle.js
 
-CERT_DIR=${LOCAL_DIR}/cert
+# Colors for the output
 
-CA_CERT_FILE=${CERT_DIR}/ca-cert.pem
-CA_KEY_FILE=${CERT_DIR}/ca-key.pem
-SERVER_CERT_REQ=${CERT_DIR}/server-req.pem
-SERVER_CERT_FILE=${CERT_DIR}/server-cert.pem
-SERVER_KEY_FILE=${CERT_DIR}/server-key.pem
-CLIENT_CERT_REQ=${CERT_DIR}/client-req.pem
-CLIENT_CERT_FILE=${CERT_DIR}/client-cert.pem
-CLIENT_KEY_FILE=${CERT_DIR}/client-key.pem
-OPENSSL_CONF=${CERT_DIR}/openssl.conf
+COLOR_GREEN=\033[0;32m
+COLOR_RESET=\033[0m
+COLOR_RED=\033[0;31m
 
-FE_BUILD_DIR=./ui/dist
-FE_BUNDLE_JS=${FE_ASSETS_DIR}/bundle.js
+# Go packages for the tools
 
-APP_NAME=dagu
-BIN_DIR=${SCRIPT_DIR}/bin
+PKG_swagger=github.com/go-swagger/go-swagger/cmd/swagger
+PKG_golangci_lint=github.com/golangci/golangci-lint/cmd/golangci-lint
+PKG_gotestsum=gotest.tools/gotestsum
+PKG_gomerger=github.com/yohamta/gomerger
 
-# gotestsum args
-GOTESTSUM_ARGS=--format=standard-quiet
-GO_TEST_FLAGS=-v --race
+# Certificates for the development environment
+
+CERTS_DIR=${LOCAL_DIR}/certs
+
+DEV_CERT_SUBJ_CA="/C=TR/ST=ASIA/L=TOKYO/O=DEV/OU=DAGU/CN=*.dagu.dev/[email protected]"
+DEV_CERT_SUBJ_SERVER="/C=TR/ST=ASIA/L=TOKYO/O=DEV/OU=SERVER/CN=*.server.dev/[email protected]"
+DEV_CERT_SUBJ_CLIENT="/C=TR/ST=ASIA/L=TOKYO/O=DEV/OU=CLIENT/CN=*.client.dev/[email protected]"
+DEV_CERT_SUBJ_ALT="subjectAltName=DNS:localhost"
+
+CA_CERT_FILE=${CERTS_DIR}/ca-cert.pem
+CA_KEY_FILE=${CERTS_DIR}/ca-key.pem
+SERVER_CERT_REQ=${CERTS_DIR}/server-req.pem
+SERVER_CERT_FILE=${CERTS_DIR}/server-cert.pem
+SERVER_KEY_FILE=${CERTS_DIR}/server-key.pem
+CLIENT_CERT_REQ=${CERTS_DIR}/client-req.pem
+CLIENT_CERT_FILE=${CERTS_DIR}/client-cert.pem
+CLIENT_KEY_FILE=${CERTS_DIR}/client-key.pem
+OPENSSL_CONF=${CONFIG_DIR}/openssl.local.conf
 
-########## Main Targets ##########
+##############################################################################
+# Targets
+##############################################################################
 
 # run starts the frontend server and the scheduler.
 run: ${FE_BUNDLE_JS}
-	go run . start-all
+	@echo "${COLOR_GREEN}Starting the frontend server and the scheduler...${COLOR_RESET}"
+	@go run . start-all
+
+# server build the binary and start the server.
+run-server: golangci-lint build-bin
+	@echo "${COLOR_GREEN}Starting the server...${COLOR_RESET}"
+	${LOCAL_BIN_DIR}/${APP_NAME} server
+
+# scheduler build the binary and start the scheduler.
+run-scheduler: golangci-lint build-bin
+	@echo "${COLOR_GREEN}Starting the scheduler...${COLOR_RESET}"
+	${LOCAL_BIN_DIR}/${APP_NAME} scheduler
 
 # check if the frontend assets are built.
 ${FE_BUNDLE_JS}:
-	echo "Please run 'make build-ui' to build the frontend assets."
+	@echo "${COLOR_RED}Error: frontend assets are not built.${COLOR_RESET}"
+	@echo "${COLOR_RED}Please run 'make build-ui' before starting the server.${COLOR_RESET}"
 
 # https starts the server with the HTTPS protocol.
-https: ${SERVER_CERT_FILE} ${SERVER_KEY_FILE}
+run-server-https: ${SERVER_CERT_FILE} ${SERVER_KEY_FILE}
+	@echo "${COLOR_GREEN}Starting the server with HTTPS...${COLOR_RESET}"
 	@DAGU_CERT_FILE=${SERVER_CERT_FILE} \
 		DAGU_KEY_FILE=${SERVER_KEY_FILE} \
 		go run . start-all
 
-# watch starts development UI server.
-# The backend server should be running.
-watch:
-	@echo "${COLOR_GREEN}Installing nodemon...${COLOR_RESET}"
-	@npm install -g nodemon
-	@nodemon --watch . --ext go,gohtml --verbose --signal SIGINT --exec 'make server'
-
 # test runs all tests.
 test:
-	@go install ${PKG_gotestsum}
-	@gotestsum ${GOTESTSUM_ARGS} -- ${GO_TEST_FLAGS} ./...
+	@echo "${COLOR_GREEN}Running tests...${COLOR_RESET}"
+	@GOBIN=${LOCAL_BIN_DIR} go install ${PKG_gotestsum}
+	@${LOCAL_BIN_DIR}/gotestsum ${GOTESTSUM_ARGS} -- ${GO_TEST_FLAGS} ./...
 
 # test-coverage runs all tests with coverage.
 test-coverage:
-	@go install ${PKG_gotestsum}
-	@gotestsum ${GOTESTSUM_ARGS} -- ${GO_TEST_FLAGS} -coverprofile="coverage.txt" -covermode=atomic ./...
+	@echo "${COLOR_GREEN}Running tests with coverage...${COLOR_RESET}"
+	@GOBIN=${LOCAL_BIN_DIR} go install ${PKG_gotestsum}
+	@${LOCAL_BIN_DIR}/gotestsum ${GOTESTSUM_ARGS} -- ${GO_TEST_FLAGS} -coverprofile="coverage.txt" -covermode=atomic ./...
 
 # test-clean cleans the test cache and run all tests.
 test-clean: build-bin
-	@go install ${PKG_gotestsum}
+	@echo "${COLOR_GREEN}Running tests...${COLOR_RESET}"
+	@GOBIN=${LOCAL_BIN_DIR} go install ${PKG_gotestsum}
 	@go clean -testcache
-	@gotestsum ${GOTESTSUM_ARGS} -- ${GO_TEST_FLAGS} ./...
+	@${LOCAL_BIN_DIR}/gotestsum ${GOTESTSUM_ARGS} -- ${GO_TEST_FLAGS} ./...
 
 # lint runs the linter.
 lint: golangci-lint
@@ -102,7 +133,7 @@ lint: golangci-lint
 swagger: clean-swagger gen-swagger
 
 # certs generates the certificates to use in the development environment.
-certs: ${SERVER_CERT_FILE} ${CLIENT_CERT_FILE} gencert-check
+certs: ${CERTS_DIR} ${SERVER_CERT_FILE} ${CLIENT_CERT_FILE} certs-check
 
 # build build the binary.
 build: build-ui build-bin
@@ -112,30 +143,38 @@ build: build-ui build-bin
 # ```sh
 # make build-image VERSION={version}
 # ```
-# {version} should be the version number such as v1.13.0.
+# {version} should be the version number such as "1.13.0".
+
 build-image: build-image-version build-image-latest
 build-image-version:
 ifeq ($(VERSION),)
-	$(error "VERSION is null")
+	$(error "VERSION is not set")
 endif
+	echo "${COLOR_GREEN}Building the docker image with the version $(VERSION)...${COLOR_RESET}"
 	$(DOCKER_CMD) -t ghcr.io/dagu-dev/${APP_NAME}:$(VERSION) .
 
 # build-image-latest build the docker image with the latest tag and push to 
 # the registry.
 build-image-latest:
+	@echo "${COLOR_GREEN}Building the docker image...${COLOR_RESET}"
 	$(DOCKER_CMD) -t ghcr.io/dagu-dev/${APP_NAME}:latest .
 
-# server build the binary and start the server.
-server: golangci-lint build-bin
-	${BIN_DIR}/${APP_NAME} server
+gomerger: ${LOCAL_DIR}/merged
+	@echo "${COLOR_GREEN}Merging Go files...${COLOR_RESET}"
+	@rm -f ${LOCAL_DIR}/merged/merged_project.go
+	@GOBIN=${LOCAL_BIN_DIR} go install ${PKG_gomerger}
+	@${LOCAL_BIN_DIR}/gomerger .
+	@mv merged_project.go ${LOCAL_DIR}/merged/
 
-# scheduler build the binary and start the scheduler.
-scheduler: golangci-lint build-bin
-	${BIN_DIR}/${APP_NAME} scheduler
+${LOCAL_DIR}/merged:
+	@mkdir -p ${LOCAL_DIR}/merged
 
-########## Tools ##########
+##############################################################################
+# Internal targets
+##############################################################################
 
 build-bin:
+	@echo "${COLOR_GREEN}Building the binary...${COLOR_RESET}"
 	@mkdir -p ${BIN_DIR}
 	@go build -ldflags="$(LDFLAGS)" -o ${BIN_DIR}/${APP_NAME} .
 
@@ -148,20 +187,25 @@ build-ui:
 	@cp ${FE_BUILD_DIR}/* ${FE_ASSETS_DIR}
 
 golangci-lint:
-	@go install $(PKG_GOLANGCI_LINT)
-	@golangci-lint run ./...
+	@echo "${COLOR_GREEN}Running linter...${COLOR_RESET}"
+	@GOBIN=${LOCAL_BIN_DIR} go install $(PKG_golangci_lint)
+	@${LOCAL_BIN_DIR}/golangci-lint run ./...
 
 clean-swagger:
+	@echo "${COLOR_GREEN}Cleaning the swagger files...${COLOR_RESET}"
 	@rm -rf ${FE_GEN_DIR}/restapi/models
 	@rm -rf ${FE_GEN_DIR}/restapi/operations
 
 gen-swagger:
-	@go install $(PKG_SWAGGER)
-	@swagger validate ./swagger.yaml
-	@swagger generate server -t ${FE_GEN_DIR} --server-package=restapi --exclude-main -f ./swagger.yaml
+	@echo "${COLOR_GREEN}Generating the swagger server code...${COLOR_RESET}"
+	@GOBIN=${LOCAL_BIN_DIR} go install $(PKG_swagger)
+	@${LOCAL_BIN_DIR}/swagger validate ./swagger.yaml
+	@${LOCAL_BIN_DIR}/swagger generate server -t ${FE_GEN_DIR} --server-package=restapi --exclude-main -f ./swagger.yaml
 	@go mod tidy
 
-########## Certificates ##########
+##############################################################################
+# Certificates
+##############################################################################
 
 ${CA_CERT_FILE}:
 	@echo "${COLOR_GREEN}Generating CA certificates...${COLOR_RESET}"
@@ -194,7 +238,10 @@ ${CLIENT_CERT_FILE}: ${CA_CERT_FILE} ${CLIENT_KEY_FILE}
 		-CAkey ${CA_KEY_FILE} -CAcreateserial -out ${CLIENT_CERT_FILE} \
 		-extfile ${OPENSSL_CONF}
 
-gencert-check:
+${CERTS_DIR}:
+	@echo "${COLOR_GREEN}Creating the certificates directory...${COLOR_RESET}"
+	@mkdir -p ${CERTS_DIR}
+
+certs-check:
 	@echo "${COLOR_GREEN}Checking CA certificate...${COLOR_RESET}"
 	@openssl x509 -in ${SERVER_CERT_FILE} -noout -text
-
diff --git a/README.md b/README.md
@@ -67,6 +67,7 @@ Dagu is a powerful Cron alternative that comes with a Web UI. It allows you to d
 -   [**Documentation**](#documentation)
 -   [**Running as a daemon**](#running-as-a-daemon)
 -   [**Example DAG**](#example-dag)
+-   [**Docker-compose setting**](#docker-compose-setting)
 -   [**Motivation**](#motivation)
 -   [**Why Not Use an Existing DAG Scheduler Like Airflow?**](#why-not-use-an-existing-dag-scheduler-like-airflow)
 -   [**How It Works**](#how-it-works)
@@ -176,12 +177,13 @@ brew upgrade dagu-dev/brew/dagu
 docker run \
 --rm \
 -p 8080:8080 \
--v $HOME/.dagu/dags:/home/dagu/.dagu/dags \
--v $HOME/.dagu/data:/home/dagu/.dagu/data \
--v $HOME/.dagu/logs:/home/dagu/.dagu/logs \
+-v $HOME/.config/dagu/dags:/home/dagu/.config/dagu/dags \
+-v $HOME/.local/share/dagu:/home/dagu/.local/share/dagu \
 ghcr.io/dagu-dev/dagu:latest dagu start-all
 ```
 
+See [Environment variables](https://dagu.readthedocs.io/en/latest/config.html#environment-variables) to configure those default directories.
+
 ## **Quick Start Guide**
 
 ### 1. Launch the Web UI
@@ -192,7 +194,7 @@ Start the server and scheduler with the command `dagu start-all` and browse to `
 
 Navigate to the DAG List page by clicking the menu in the left panel of the Web UI. Then create a DAG by clicking the `NEW` button at the top of the page. Enter `example` in the dialog.
 
-_Note: DAG (YAML) files will be placed in `~/.dagu/dags` by default. See [Configuration Options](https://dagu.readthedocs.io/en/latest/config.html) for more details._
+_Note: DAG (YAML) files will be placed in `~/.config/dagu/dags` by default. See [Configuration Options](https://dagu.readthedocs.io/en/latest/config.html) for more details._
 
 ### 3. Edit the DAG
 
@@ -387,6 +389,10 @@ steps:
           - send_report
 ```
 
+## **Docker-compose setting**
+
+To run Dagu using Docker-compose, please take a look at the example: [docker-compose file](examples/docker-compose.yaml)
+
 ## **Motivation**
 
 Legacy systems often have complex and implicit dependencies between jobs. When there are hundreds of cron jobs on a server, it can be difficult to keep track of these dependencies and to determine which job to rerun if one fails. It can also be a hassle to SSH into a server to view logs and manually rerun shell scripts one by one. Dagu aims to solve these problems by allowing you to explicitly visualize and manage pipeline dependencies as a DAG, and by providing a web UI for checking dependencies, execution status, and logs and for rerunning or stopping jobs with a simple mouse click.