Qwen2.5 VL 模型部署

本章节包含 Qwen2.5 VL 系列模型的部署脚本。

• 模型权重下载。

• 参考 环境准备 进行环境检查、启动容器。

• 容器镜像地址参考 版本发布信息-镜像地址。

Qwen2.5-VL-7B

本示例使用 1卡（TP1） 部署。

启动脚本 start_server.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export SGLANG_TORCH_PROFILER_DIR=/tmp/traces
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
export SGLANG_USE_CUDA_IPC_TRANSPORT="${SGLANG_USE_CUDA_IPC_TRANSPORT:-1}"
export SGLANG_USE_IPC_POOL_HANDLE_CACHE="${SGLANG_USE_IPC_POOL_HANDLE_CACHE:-1}"

# pkill -f "sgl.*"
# sleep 3

CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
SGLANG_PORT="${SGLANG_PORT:-31000}"
MASTER_IP=127.0.0.1
NODE_RANK=0
WORLD_SIZE=1
MODEL_PATH=/data/models/Qwen2.5-VL-7B-Instruct/
LOG_DIR="$WORK_HOME/output/$CURRENT_TIME"
LOG_FILE="${LOG_DIR}/sglang_$(date '+%Y%m%d_%H%M%S').log"

mkdir -p "$LOG_DIR"

nohup "$SGLANG_PYTHON" -m sglang.launch_server \
    --model-path "${MODEL_PATH}" \
    --trust-remote-code \
    --cuda-graph-bs $(seq 1 90) \
    --disable-overlap-schedule \
    --tp-size 1 \
    --pp-size 1 \
    --linear-attn-backend flashinfer \
    --tokenizer-backend fastokens \
    --mem-fraction-static 0.85 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --moe-runner-backend auto \
    --sampling-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:50022 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size -1 \
    --port ${SGLANG_PORT} \
    --host "${SGLANG_HOST}" \
    --decode-log-interval 1 > "$LOG_FILE" 2>&1 &

Qwen2.5-VL-32B

本示例使用 4卡（TP4） 部署。

启动脚本 start_server.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export SGLANG_TORCH_PROFILER_DIR=/tmp/traces
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
export SGLANG_USE_CUDA_IPC_TRANSPORT="${SGLANG_USE_CUDA_IPC_TRANSPORT:-1}"
export SGLANG_USE_IPC_POOL_HANDLE_CACHE="${SGLANG_USE_IPC_POOL_HANDLE_CACHE:-1}"

# pkill -f "sgl.*"
# sleep 3

CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
SGLANG_PORT="${SGLANG_PORT:-31000}"
MASTER_IP=127.0.0.1
NODE_RANK=0
WORLD_SIZE=1
MODEL_PATH=/data/models/Qwen2.5-VL-32B-Instruct/
LOG_DIR="$WORK_HOME/output/$CURRENT_TIME"
LOG_FILE="${LOG_DIR}/sglang_$(date '+%Y%m%d_%H%M%S').log"

mkdir -p "$LOG_DIR"

nohup "$SGLANG_PYTHON" -m sglang.launch_server \
    --model-path "${MODEL_PATH}" \
    --trust-remote-code \
    --cuda-graph-bs $(seq 1 128) \
    --disable-overlap-schedule \
    --tp-size 4 \
    --pp-size 1 \
    --mem-fraction-static 0.8 \
    --attention-backend fa3 \
    --linear-attn-backend flashinfer \
    --tokenizer-backend fastokens \
    --mm-attention-backend fa3 \
    --moe-runner-backend auto \
    --sampling-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:50022 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size -1 \
    --port ${SGLANG_PORT} \
    --host "${SGLANG_HOST}" \
    --decode-log-interval 1 > "$LOG_FILE" 2>&1 &

Qwen2.5-VL-72B

本示例使用单机 8卡（TP8） 部署。

启动脚本 start_server.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export SGLANG_TORCH_PROFILER_DIR=/tmp/traces
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
export SGLANG_USE_CUDA_IPC_TRANSPORT="${SGLANG_USE_CUDA_IPC_TRANSPORT:-1}"
export SGLANG_USE_IPC_POOL_HANDLE_CACHE="${SGLANG_USE_IPC_POOL_HANDLE_CACHE:-1}"

# pkill -f "sgl.*"
# sleep 3

CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
SGLANG_PORT="${SGLANG_PORT:-31000}"
MASTER_IP=127.0.0.1
NODE_RANK=0
WORLD_SIZE=1
MODEL_PATH=/data/models/Qwen2.5-VL-72B-Instruct/
LOG_DIR="$WORK_HOME/output/$CURRENT_TIME"
LOG_FILE="${LOG_DIR}/sglang_$(date '+%Y%m%d_%H%M%S').log"


mkdir -p "$LOG_DIR"

nohup "$SGLANG_PYTHON" -m sglang.launch_server \
    --model-path "${MODEL_PATH}" \
    --trust-remote-code \
    --cuda-graph-bs $(seq 1 256) \
    --disable-overlap-schedule \
    --tp-size 8 \
    --pp-size 1 \
    --mem-fraction-static 0.85 \
    --linear-attn-backend flashinfer \
    --tokenizer-backend fastokens \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --moe-runner-backend auto \
    --sampling-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:50022 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size -1 \
    --port ${SGLANG_PORT} \
    --host "${SGLANG_HOST}" \
    --decode-log-interval 1 > "$LOG_FILE" 2>&1 &