Improved image build instructions (infiniflow#3580)

yuzhichang · web-flow · commit 646ac1f2b46a · 2024-11-22T20:24:32.000+08:00
### What problem does this PR solve?

Improved arm64 image build instructions

### Type of change

- [x] Documentation Update
- [x] Refactoring
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -48,13 +48,13 @@ jobs:
       - name: Build ragflow:dev-slim
         run: |
           RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME}
-          cp -r ${RUNNER_WORKSPACE_PREFIX}/huggingface.co ${RUNNER_WORKSPACE_PREFIX}/nltk_data ${RUNNER_WORKSPACE_PREFIX}/libssl*.deb  ${RUNNER_WORKSPACE_PREFIX}/tika-server*.jar* .
-          sudo docker pull ubuntu:24.04
-          sudo ./build_docker_image.sh slim
+          cp -r ${RUNNER_WORKSPACE_PREFIX}/huggingface.co ${RUNNER_WORKSPACE_PREFIX}/nltk_data ${RUNNER_WORKSPACE_PREFIX}/libssl*.deb  ${RUNNER_WORKSPACE_PREFIX}/tika-server*.jar* ${RUNNER_WORKSPACE_PREFIX}/chrome* ${RUNNER_WORKSPACE_PREFIX}/cl100k_base.tiktoken .
+          sudo docker pull ubuntu:22.04
+          sudo docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
 
       - name: Build ragflow:dev
         run: |
-          sudo ./build_docker_image.sh full
+          sudo docker build -f Dockerfile -t infiniflow/ragflow:dev .
 
       - name: Start ragflow:dev-slim
         run: |
diff --git a/Dockerfile b/Dockerfile
@@ -1,8 +1,8 @@
 # base stage
 FROM ubuntu:22.04 AS base
 USER root
+SHELL ["/bin/bash", "-c"]
 
-ARG ARCH=amd64
 ENV LIGHTEN=0
 
 WORKDIR /ragflow
@@ -18,7 +18,7 @@ RUN sed -i 's|http://archive.ubuntu.com|https://mirrors.tuna.tsinghua.edu.cn|g'
 
 RUN --mount=type=cache,id=ragflow_base_apt,target=/var/cache/apt,sharing=locked \
     apt update && DEBIAN_FRONTEND=noninteractive apt install -y curl libpython3-dev nginx libglib2.0-0 libglx-mesa0 pkg-config libicu-dev libgdiplus default-jdk python3-pip pipx \
-    libatk-bridge2.0-0 libgtk-4-1 libnss3 xdg-utils unzip libgbm-dev wget \
+    libatk-bridge2.0-0 libgtk-4-1 libnss3 xdg-utils unzip libgbm-dev wget git \
     && rm -rf /var/lib/apt/lists/*
 
 RUN pip3 config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple && pip3 config set global.trusted-host "pypi.tuna.tsinghua.edu.cn mirrors.pku.edu.cn" && pip3 config set global.extra-index-url "https://mirrors.pku.edu.cn/pypi/web/simple" \
@@ -28,8 +28,11 @@ RUN pip3 config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple &&
 # https://forum.aspose.com/t/aspose-slides-for-net-no-usable-version-of-libssl-found-with-linux-server/271344/13
 # aspose-slides on linux/arm64 is unavailable
 RUN --mount=type=bind,source=libssl1.1_1.1.1f-1ubuntu2_amd64.deb,target=/root/libssl1.1_1.1.1f-1ubuntu2_amd64.deb \
-    if [ "${ARCH}" = "amd64" ]; then \
+    --mount=type=bind,source=libssl1.1_1.1.1f-1ubuntu2_arm64.deb,target=/root/libssl1.1_1.1.1f-1ubuntu2_arm64.deb \
+    if [ "$(uname -m)" = "x86_64" ]; then \
         dpkg -i /root/libssl1.1_1.1.1f-1ubuntu2_amd64.deb; \
+    elif [ "$(uname -m)" = "aarch64" ]; then \
+        dpkg -i /root/libssl1.1_1.1.1f-1ubuntu2_arm64.deb; \
     fi
 
 ENV PYTHONDONTWRITEBYTECODE=1 DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1
@@ -56,6 +59,24 @@ USER root
 
 WORKDIR /ragflow
 
+COPY .git /ragflow/.git
+
+RUN current_commit=$(git rev-parse --short HEAD); \
+    last_tag=$(git describe --tags --abbrev=0); \
+    commit_count=$(git rev-list --count "$last_tag..HEAD"); \
+    version_info=""; \
+    if [ "$commit_count" -eq 0 ]; then \
+        version_info=$last_tag; \
+    else \
+        version_info="$current_commit($last_tag~$commit_count)"; \
+    fi; \
+    if [ "$LIGHTEN" == "1" ]; then \
+        version_info="$version_info slim"; \
+    else \
+        version_info="$version_info full"; \
+    fi; \
+    echo $version_info > /ragflow/VERSION
+
 COPY web web
 COPY docs docs
 RUN --mount=type=cache,id=ragflow_builder_npm,target=/root/.npm,sharing=locked \
@@ -65,10 +86,10 @@ RUN --mount=type=cache,id=ragflow_builder_npm,target=/root/.npm,sharing=locked \
 COPY pyproject.toml poetry.toml poetry.lock ./
 
 RUN --mount=type=cache,id=ragflow_builder_poetry,target=/root/.cache/pypoetry,sharing=locked \
-    if [ "$LIGHTEN" -eq 0 ]; then \
-        poetry install --no-root --with=full; \
-    else \
+    if [ "$LIGHTEN" == "1" ]; then \
         poetry install --no-root; \
+    else \
+        poetry install --no-root --with=full; \
     fi
 
 # production stage
@@ -77,6 +98,8 @@ USER root
 
 WORKDIR /ragflow
 
+COPY --from=builder /ragflow/VERSION /ragflow/VERSION
+
 # Install python packages' dependencies
 # cv2 requires libGL.so.1
 RUN --mount=type=cache,id=ragflow_production_apt,target=/var/cache/apt,sharing=locked \
diff --git a/Dockerfile.slim b/Dockerfile.slim
@@ -1,8 +1,8 @@
 # base stage
 FROM ubuntu:22.04 AS base
 USER root
+SHELL ["/bin/bash", "-c"]
 
-ARG ARCH=amd64
 ENV LIGHTEN=1
 
 WORKDIR /ragflow
@@ -18,7 +18,7 @@ RUN sed -i 's|http://archive.ubuntu.com|https://mirrors.tuna.tsinghua.edu.cn|g'
 
 RUN --mount=type=cache,id=ragflow_base_apt,target=/var/cache/apt,sharing=locked \
     apt update && DEBIAN_FRONTEND=noninteractive apt install -y curl libpython3-dev nginx libglib2.0-0 libglx-mesa0 pkg-config libicu-dev libgdiplus default-jdk python3-pip pipx \
-    libatk-bridge2.0-0 libgtk-4-1 libnss3 xdg-utils unzip libgbm-dev wget \
+    libatk-bridge2.0-0 libgtk-4-1 libnss3 xdg-utils unzip libgbm-dev wget git \
     && rm -rf /var/lib/apt/lists/*
 
 RUN pip3 config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple && pip3 config set global.trusted-host "pypi.tuna.tsinghua.edu.cn mirrors.pku.edu.cn" && pip3 config set global.extra-index-url "https://mirrors.pku.edu.cn/pypi/web/simple" \
@@ -28,8 +28,11 @@ RUN pip3 config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple &&
 # https://forum.aspose.com/t/aspose-slides-for-net-no-usable-version-of-libssl-found-with-linux-server/271344/13
 # aspose-slides on linux/arm64 is unavailable
 RUN --mount=type=bind,source=libssl1.1_1.1.1f-1ubuntu2_amd64.deb,target=/root/libssl1.1_1.1.1f-1ubuntu2_amd64.deb \
-    if [ "${ARCH}" = "amd64" ]; then \
+    --mount=type=bind,source=libssl1.1_1.1.1f-1ubuntu2_arm64.deb,target=/root/libssl1.1_1.1.1f-1ubuntu2_arm64.deb \
+    if [ "$(uname -m)" = "x86_64" ]; then \
         dpkg -i /root/libssl1.1_1.1.1f-1ubuntu2_amd64.deb; \
+    elif [ "$(uname -m)" = "aarch64" ]; then \
+        dpkg -i /root/libssl1.1_1.1.1f-1ubuntu2_arm64.deb; \
     fi
 
 ENV PYTHONDONTWRITEBYTECODE=1 DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1
@@ -56,6 +59,24 @@ USER root
 
 WORKDIR /ragflow
 
+COPY .git /ragflow/.git
+
+RUN current_commit=$(git rev-parse --short HEAD); \
+    last_tag=$(git describe --tags --abbrev=0); \
+    commit_count=$(git rev-list --count "$last_tag..HEAD"); \
+    version_info=""; \
+    if [ "$commit_count" -eq 0 ]; then \
+        version_info=$last_tag; \
+    else \
+        version_info="$current_commit($last_tag~$commit_count)"; \
+    fi; \
+    if [ "$LIGHTEN" == "1" ]; then \
+        version_info="$version_info slim"; \
+    else \
+        version_info="$version_info full"; \
+    fi; \
+    echo $version_info > /ragflow/VERSION
+
 COPY web web
 COPY docs docs
 RUN --mount=type=cache,id=ragflow_builder_npm,target=/root/.npm,sharing=locked \
@@ -65,10 +86,10 @@ RUN --mount=type=cache,id=ragflow_builder_npm,target=/root/.npm,sharing=locked \
 COPY pyproject.toml poetry.toml poetry.lock ./
 
 RUN --mount=type=cache,id=ragflow_builder_poetry,target=/root/.cache/pypoetry,sharing=locked \
-    if [ "$LIGHTEN" -eq 0 ]; then \
-        poetry install --no-root --with=full; \
-    else \
+    if [ "$LIGHTEN" == "1" ]; then \
         poetry install --no-root; \
+    else \
+        poetry install --no-root --with=full; \
     fi
 
 # production stage
@@ -77,6 +98,8 @@ USER root
 
 WORKDIR /ragflow
 
+COPY --from=builder /ragflow/VERSION /ragflow/VERSION
+
 # Install python packages' dependencies
 # cv2 requires libGL.so.1
 RUN --mount=type=cache,id=ragflow_production_apt,target=/var/cache/apt,sharing=locked \
diff --git a/README.md b/README.md
@@ -274,7 +274,7 @@ git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
 pip3 install huggingface-hub nltk
 python3 download_deps.py
-bash build_docker_image.sh slim
+docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
 ```
 
 ## 🔧 Build a Docker image including embedding models
@@ -286,7 +286,7 @@ git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
 pip3 install huggingface-hub nltk
 python3 download_deps.py
-bash build_docker_image.sh full
+docker build -f Dockerfile -t infiniflow/ragflow:dev .
 ```
 
 ## 🔨 Launch service from source for development
diff --git a/README_id.md b/README_id.md
@@ -249,7 +249,7 @@ git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
 pip3 install huggingface-hub nltk
 python3 download_deps.py
-bash build_docker_image.sh slim
+docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
 ```
 
 ## 🔧 Membangun Docker Image Termasuk Model Embedding
@@ -261,7 +261,7 @@ git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
 pip3 install huggingface-hub nltk
 python3 download_deps.py
-bash build_docker_image.sh full
+docker build -f Dockerfile -t infiniflow/ragflow:dev .
 ```
 
 ## 🔨 Menjalankan Aplikasi dari untuk Pengembangan
diff --git a/README_ja.md b/README_ja.md
@@ -230,7 +230,7 @@ git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
 pip3 install huggingface-hub nltk
 python3 download_deps.py
-bash build_docker_image.sh slim
+docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
 ```
 
 ## 🔧 ソースコードをコンパイルしたDockerイメージ（埋め込みモデルを含む）
@@ -242,7 +242,7 @@ git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
 pip3 install huggingface-hub nltk
 python3 download_deps.py
-bash build_docker_image.sh full
+docker build -f Dockerfile -t infiniflow/ragflow:dev .
 ```
 
 ## 🔨 ソースコードからサービスを起動する方法
diff --git a/README_ko.md b/README_ko.md
@@ -232,7 +232,7 @@ git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
 pip3 install huggingface-hub nltk
 python3 download_deps.py
-bash build_docker_image.sh slim
+docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
 ```
 
 ## 🔧 소스 코드로 Docker 이미지를 컴파일합니다(임베딩 모델 포함)
@@ -244,7 +244,7 @@ git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
 pip3 install huggingface-hub nltk
 python3 download_deps.py
-bash build_docker_image.sh full
+docker build -f Dockerfile -t infiniflow/ragflow:dev .
 ```
 
 ## 🔨 소스 코드로 서비스를 시작합니다.
diff --git a/README_zh.md b/README_zh.md
@@ -237,7 +237,7 @@ git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
 pip3 install huggingface-hub nltk
 python3 download_deps.py
-bash build_docker_image.sh slim
+docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
 ```
 
 ## 🔧 源码编译 Docker 镜像（包含 embedding 模型）
@@ -249,7 +249,7 @@ git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
 pip3 install huggingface-hub nltk
 python3 download_deps.py
-bash build_docker_image.sh full
+docker build -f Dockerfile -t infiniflow/ragflow:dev .
 ```
 
 ## 🔨 以源代码启动服务
diff --git a/api/versions.py b/api/versions.py
@@ -13,26 +13,51 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+
+import os
 import subprocess
 
-def get_ragflow_version() -> str:
-    return RAGFLOW_VERSION_INFO
+RAGFLOW_VERSION_INFO = "unknown"
 
 
-RAGFLOW_VERSION_INFO = "dev"
+def get_ragflow_version() -> str:
+    global RAGFLOW_VERSION_INFO
+    if RAGFLOW_VERSION_INFO != "unknown":
+        return RAGFLOW_VERSION_INFO
+    version_path = os.path.abspath(
+        os.path.join(
+            os.path.dirname(os.path.realpath(__file__)), os.pardir, "VERSION"
+        )
+    )
+    if os.path.exists(version_path):
+        with open(version_path, "r") as f:
+            RAGFLOW_VERSION_INFO = f.read().strip()
+    else:
+        RAGFLOW_VERSION_INFO = get_closest_tag_and_count()
+        LIGHTEN = int(os.environ.get("LIGHTEN", "0"))
+        RAGFLOW_VERSION_INFO += " slim" if LIGHTEN == 1 else " full"
+    return RAGFLOW_VERSION_INFO
 
 
 def get_closest_tag_and_count():
     try:
         # Get the current commit hash
-        commit_id = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).strip().decode('utf-8')
+        commit_id = (
+            subprocess.check_output(["git", "rev-parse", "--short", "HEAD"])
+            .strip()
+            .decode("utf-8")
+        )
         # Get the closest tag
-        closest_tag = subprocess.check_output(['git', 'describe', '--tags', '--abbrev=0']).strip().decode('utf-8')
-        # Get the commit hash of the closest tag
-        closest_tag_commit = subprocess.check_output(['git', 'rev-list', '-n', '1', closest_tag]).strip().decode(
-            'utf-8')
+        closest_tag = (
+            subprocess.check_output(["git", "describe", "--tags", "--abbrev=0"])
+            .strip()
+            .decode("utf-8")
+        )
         # Get the commit count since the closest tag
-        process = subprocess.Popen(['git', 'rev-list', '--count', f'{closest_tag}..HEAD'], stdout=subprocess.PIPE)
+        process = subprocess.Popen(
+            ["git", "rev-list", "--count", f"{closest_tag}..HEAD"],
+            stdout=subprocess.PIPE,
+        )
         commits_count, _ = process.communicate()
         commits_count = int(commits_count.strip())
 
@@ -41,8 +66,4 @@ def get_closest_tag_and_count():
         else:
             return f"{commit_id}({closest_tag}~{commits_count})"
     except Exception:
-        return 'unknown'
-
-
-if RAGFLOW_VERSION_INFO == 'dev':
-    RAGFLOW_VERSION_INFO = get_closest_tag_and_count()
+        return "unknown"
diff --git a/build_docker_image.sh b/build_docker_image.sh
diff --git a/docs/guides/develop/build_docker_image.mdx b/docs/guides/develop/build_docker_image.mdx
@@ -73,8 +73,8 @@ git clone https://github.com/infiniflow/ragflow.git
 cd ragflow/
 pip3 install huggingface-hub nltk
 python3 download_deps.py
-docker build --build-arg ARCH=arm64 -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
-docker build --build-arg ARCH=arm64 -f Dockerfile -t infiniflow/ragflow:dev .
+docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
+docker build -f Dockerfile -t infiniflow/ragflow:dev .
 ```
   </TabItem>
 
diff --git a/download_deps.py b/download_deps.py
