Qwen2.5 VL 模型部署

本章节包含 Qwen2.5 VL 系列模型的部署脚本，脚本涉及的参数说明见 快速开始。

信息

启动 SGLang 服务前切换python虚拟环境到sglang-0.5.6，执行命令：workon sglang-0.5.6 或 source ~/.virtualenvs/sglang-0.5.6/bin/activate

Qwen2.5-VL-7B

本示例使用 1卡（TP1） 部署。

启动脚本 qwen25_vl_7b_server.sh

#!/bin/bash
source ~/.virtualenvs/sglang-0.5.6/bin/activate

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export SGLANG_TORCH_PROFILER_DIR=/tmp/traces
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1

WORK_HOME="$PWD"
SGLANG_DECODE_PORT=20133
MASTER_IP=127.0.0.1
NODE_RANK=0
WORLD_SIZE=1
MODEL_PATH=/data/models/Qwen2.5-VL-7B-Instruct/
LOG_DIR=$WORK_HOME/output/$CURRENT_TIME
PYTHON="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-0.5.6}/bin/python3"

nohup "${PYTHON}" -m sglang.launch_server \
    --model ${MODEL_PATH} \
    --served-model-name qwen2.5-vl-7b-instruct \
    --trust-remote-code \
    --cuda-graph-bs $(seq 1 90) \
    --disable-cuda-graph-padding \
    --disable-overlap-schedule \
    --tp-size 1 \
    --pp-size 1 \
    --mem-fraction-static 0.85 \
    --tool-call-parser qwen25 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --moe-runner-backend deep_gemm \
    --sampling-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:5303 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size -1 \
    --disable-radix-cache \
    --port ${SGLANG_DECODE_PORT} \
    --host 0.0.0.0 \
    --decode-log-interval 1 > "${LOG_DIR}/$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

Qwen2.5-VL-32B

本示例使用 4卡（TP4） 部署。

启动脚本 qwen25_vl_32b_server.sh

#!/bin/bash
source ~/.virtualenvs/sglang-0.5.6/bin/activate

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export SGLANG_TORCH_PROFILER_DIR=/tmp/traces
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1

WORK_HOME="$PWD"
SGLANG_DECODE_PORT=20133
MASTER_IP=127.0.0.1
NODE_RANK=0
WORLD_SIZE=1
MODEL_PATH=/data/models/Qwen2.5-VL-32B-Instruct/
LOG_DIR=$WORK_HOME/output/$CURRENT_TIME
PYTHON="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-0.5.6}/bin/python3"


nohup "${PYTHON}" -m sglang.launch_server \
    --model ${MODEL_PATH} \
    --tokenizer-path ${MODEL_PATH} \
    --served-model-name qwen2.5-vl-32b-instruct \
    --trust-remote-code \
    --cuda-graph-bs $(seq 1 128) \
    --disable-cuda-graph-padding \
    --disable-overlap-schedule \
    --tp-size 4 \
    --pp-size 1 \
    --mem-fraction-static 0.8 \
    --attention-backend fa3 \
    --tool-call-parser qwen25 \
    --mm-attention-backend fa3 \
    --moe-runner-backend deep_gemm \
    --sampling-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:5303 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size -1 \
    --disable-radix-cache \
    --port ${SGLANG_DECODE_PORT} \
    --host 0.0.0.0 \
    --decode-log-interval 1 > "${LOG_DIR}/$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

Qwen2.5-VL-72B

本示例使用单机 8卡（TP8） 部署。

启动脚本 qwen25_vl_72b_server.sh

#!/bin/bash
source ~/.virtualenvs/sglang-0.5.6/bin/activate

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export SGLANG_TORCH_PROFILER_DIR=/tmp/traces
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1

WORK_HOME="$PWD"
SGLANG_DECODE_PORT=20133
MASTER_IP=127.0.0.1
NODE_RANK=0
WORLD_SIZE=1
MODEL_PATH=/data/models/Qwen2.5-VL-72B-Instruct/
LOG_DIR=$WORK_HOME/output/$CURRENT_TIME
PYTHON="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-0.5.6}/bin/python3"

nohup "${PYTHON}" -m sglang.launch_server \
    --model ${MODEL_PATH} \
    --served-model-name qwen2.5-vl-72b-instruct \
    --trust-remote-code \
    --cuda-graph-bs $(seq 1 256) \
    --disable-cuda-graph-padding \
    --disable-overlap-schedule \
    --tp-size 8 \
    --pp-size 1 \
    --mem-fraction-static 0.85 \
    --tool-call-parser qwen25 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --moe-runner-backend deep_gemm \
    --sampling-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:5303 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size -1 \
    --disable-radix-cache \
    --port ${SGLANG_DECODE_PORT} \
    --host 0.0.0.0 \
    --decode-log-interval 1 > "${LOG_DIR}/$(date '+%Y%m%d_%H%M%S').log" 2>&1 &