FROM rocm/pytorch

WORKDIR /root/
ARG ROCM_TARGET_LST=/root/gfx
RUN echo "gfx1100" > /root/gfx
RUN rocm_agent_enumerator

RUN git clone https://github.com/ggerganov/llama.cpp.git
WORKDIR /root/llama.cpp

RUN make GGML_HIPBLAS=1 -j$(nproc)

ENTRYPOINT ["bash"]

FROM rocm/pytorch

WORKDIR /root/
ARG ROCM_TARGET_LST=/root/gfx
RUN echo "gfx1100" > /root/gfx
RUN rocm_agent_enumerator

RUN git clone --recurse-submodules https://github.com/abetlen/llama-cpp-python.git
WORKDIR /root/llama-cpp-python

ARG CMAKE_ARGS="-DGGML_HIPBLAS=ON -DAMDGPU_TARGETS=gfx1100 -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++"
RUN pip install --upgrade pip
RUN pip install -e .
RUN pip install -e .[server]

ENTRYPOINT ["python3", "-m", "llama_cpp.server", "--config_file", "/root/models.conf"]

{
    "host": "0.0.0.0",
    "port": 8000,
    "models": [
        {
            "model": "/models/Qwen2.5-Coder-32B-Instruct-Q4_K_S.gguf",
            "model_alias": "Qwen2.5-Coder-32B",
            "n_gpu_layers": -1,
            "offload_kqv": true
        }
    ]
}

llm_load_tensors: tensor 'token_embd.weight' (q4_K) (and 0 others) cannot be used with preferred buffer type ROCm_Host, using CPU instead
llm_load_tensors: offloading 64 repeating layers to GPU
llm_load_tensors: offloading output layer to GPU
llm_load_tensors: offloaded 65/65 layers to GPU
llm_load_tensors: CPU_Mapped model buffer size =   417.66 MiB
llm_load_tensors:      ROCm0 model buffer size = 17490.85 MiB
warning: failed to mlock 1082613760-byte buffer (after previously locking 0 bytes): Cannot allocate memory
Try increasing RLIMIT_MEMLOCK ('ulimit -l' as root).