Qwen3 VL 模型部署

本章节包含 Qwen3 VL 系列模型的部署脚本，脚本涉及的参数说明、PD 分离一键部署说明见 快速开始。

信息

启动 SGLang 服务前切换python虚拟环境到sglang-0.5.6，执行命令：workon sglang-0.5.6 或 source ~/.virtualenvs/sglang-0.5.6/bin/activate

Qwen3VL-2/4/8B

本示例使用 1卡（TP1）部署 Qwen3-VL-2B-Instruct。 对于 Qwen3-VL-4B-Instruct 和 Qwen3-VL-8B-Instruct，修改 MODEL_PATH 环境变量和 served-model-name 参数即可。

启动脚本 Launch_server.sh

#!/bin/bash
source ~/.virtualenvs/sglang-0.5.6/bin/activate

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export SGLANG_TORCH_PROFILER_DIR=/tmp/traces
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1

SGLANG_DECODE_PORT=20133
MASTER_IP=127.0.0.1
NODE_RANK=0
WORLD_SIZE=1
MODEL_PATH=/data/models/Qwen3-VL-2B-Instruct-FP8/
log_path=logs-sglang-server/$(hostname)-$(basename $MODEL_PATH)
mkdir -p $log_path
log_file=$(date "+%Y%m%d_%H%M").log
touch $log_path/$log_file
rm -f $log_path/latest
ln -s $log_file $log_path/latest
export MUSA_VISIBLE_DEVICES=$run_GPU
python3 -m sglang.launch_server \
    --model ${MODEL_PATH} \
    --served-model-name qwen3-vl-2b-instruct-fp8 \
    --trust-remote-code \
    --cuda-graph-max-bs 256 \
    --disable-overlap-schedule \
    --tp-size 1 \
    --pp-size 1 \
    --mem-fraction-static 0.8 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --moe-runner-backend deep_gemm \
    --sampling-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:5303 \
    --nnodes 1 \
    --node-rank 0 \
    --max-prefill-tokens 4096 \
    --chunked-prefill-size -1 \
    --disable-radix-cache \
    --port ${SGLANG_DECODE_PORT} \
    --host 127.0.0.1 \
    --decode-log-interval 1 2>&1 | tee $log_path/$log_file

Qwen3VL-32B

本示例使用 4卡（TP4）部署。

启动脚本 qwen3_vl_32b_server.sh

#!/bin/bash
source ~/.virtualenvs/sglang-0.5.6/bin/activate

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export SGLANG_TORCH_PROFILER_DIR=/tmp/traces
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1
export LD_LIBRARY_PATH=/usr/local/musa/lib:${LD_LIBRARY_PATH}

SGLANG_DECODE_PORT=20133
MASTER_IP=127.0.0.1
NODE_RANK=0
WORLD_SIZE=1
MODEL_PATH=/data/models/Qwen3-VL-32B-Instruct-FP8/
log_path=logs-sglang-server/$(hostname)
mkdir -p $log_path
log_file=$(date "+%Y%m%d_%H%M").log
touch $log_path/$log_file
rm -f $log_path/latest
ln -s $log_file $log_path/latest
python3 -m sglang.launch_server \
    --model ${MODEL_PATH} \
    --served-model-name qwen3-vl-32b-instruct-fp8 \
    --trust-remote-code \
    --cuda-graph-max-bs 256 \
    --disable-overlap-schedule \
    --tp-size 4 \
    --dp-size 1 \
    --mem-fraction-static 0.8 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:5303 \
    --nnodes 1 \
    --node-rank 0 \
    --chunked-prefill-size -1 \
    --disable-radix-cache \
    --port ${SGLANG_DECODE_PORT} \
    --host 0.0.0.0 \
    --decode-log-interval 1 2>&1 | tee $log_path/$log_file

Qwen3-VL-30B-A3B-Instruct

本示例使用 2卡（TP2）部署。

Qwen3-VL-30B-A3B-Instruct 启动脚本 qwen3_vl_30b_a3b_server.sh

#!/bin/bash
source ~/.virtualenvs/sglang-0.5.6/bin/activate

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1

get_master_ip() {
    local hostname_ip=$(hostname -I 2>/dev/null | awk '{print $1}')
  
    if [ -n "$hostname_ip" ] && [ "$hostname_ip" != "127.0.0.1" ]; then
        echo "$hostname_ip"
        return
    fi
  
}

CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
SGLANG_DECODE_PORT=20133
MASTER_IP=$(get_master_ip)
NODE_RANK=0
WORLD_SIZE=1
WORK_HOME="$PWD"
LOG_DIR=$WORK_HOME/output/$CURRENT_TIME/
mkdir -p "$LOG_DIR"

export MUSA_VISIBLE_DEVICES=0,1
export SGLANG_TORCH_PROFILER_DIR=$WORK_HOME/traces
MODEL_PATH=/data/models/Qwen3-VL-30B-A3B-Instruct-FP8/
nohup python3 -m sglang.launch_server \
      --model-path ${MODEL_PATH} \
      --served-model-name qwen3-vl-30b-a3b-instruct-fp8 \
      --trust-remote-code \
      --disable-overlap-schedule \
      --cuda-graph-max-bs 512 \
      --tp-size 2 \
      --ep-size 2 \
      --pp-size 1 \
      --mem-fraction-static 0.8 \
      --attention-backend fa3 \
      --enable-cache-report \
      --mm-attention-backend fa3 \
      --moe-runner-backend deep_gemm \
      --enable-dp-attention \
      --enable-dp-lm-head \
      --moe-dense-tp-size 1 \
      --sampling-backend flashinfer \
      --dist-init-addr ${MASTER_IP}:5303 \
      --nnodes ${WORLD_SIZE} \
      --node-rank ${NODE_RANK} \
      --chunked-prefill-size -1 \
      --max-prefill-tokens 4096 \
      --context-length 16384 \
      --disable-radix-cache \
      --port ${SGLANG_DECODE_PORT} \
      --host 0.0.0.0 \
      --decode-log-interval 1 > "${LOG_DIR}/${MASTER_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

echo -e "Main log file: \033[34m$LOG_DIR\033[0m"

Qwen3-VL-235B-A22B-Instruct

本示例部署方式为1P1D，请准备两台机器。模型使用FP8，如需运行BF16版本，修改 MODEL_PATH 环境变量和 served-model-name 参数即可。

Prefill启动脚本 prefill_server.sh

#!/bin/bash
source ~/.virtualenvs/sglang-0.5.6/bin/activate

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export PATH=/root/.virtualenvs/sglang-0.5.6/bin/:$PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0

export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEP_GEMM_BLOCK_M=256
# 如果模型为 BF16 数据类型则打开该环境变量
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true

export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600

export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1

# 获取当前环境的 RDMA 设备名列表，在启动时作为 disaggregation-ib-device 参数值
IB_DEVS=()
           
for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        # 速率（如 200 Gb/sec）
        rate=$(cat "$port/rate" 2>/dev/null)
        # link_layer: InfiniBand / Ethernet (RoCE)
        link=$(cat "$port/link_layer" 2>/dev/null)
        # 状态为 Up
        state=$(cat "$port/state" 2>/dev/null)
        # 只要是 200G/400G 且状态为 Up（IB 或 RoCE 都收）
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]] && [[ "$state" == "4"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
# 排序 + 去重
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
# 生成 MCCL / SGLang 变量
MCCL_IB_HCA=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
SGLANG_IB_DEVICES="$MCCL_IB_HCA"
echo "IB_DEVICES=$MCCL_IB_HCA"
# 获取 RDMA 设备名列表结束

NODE_RANK=$1
PREFILL_IP="${2}:5303"
LOG_DIR=$3
MODEL_PATH=$4
HOST_IP=$5

WORLD_SIZE=1
WORK_HOME="$PWD"
export SGLANG_TORCH_PROFILER_DIR=$WORK_HOME/P${NODE_RANK}_traces

mkdir -p "${LOG_DIR}"

nohup python3 -m sglang.launch_server \
--model-path ${MODEL_PATH} \
      --served-model-name qwen3-vl-235b-a22b-instruct-fp8 \
      --trust-remote-code \
      --disable-overlap-schedule \
      --disable-cuda-graph \
      --tp-size 8 \
      --dp-size 8 \
      --ep-size 8 \
      --pp-size 1 \
      --mem-fraction-static 0.85 \
      --attention-backend fa3 \
      --enable-cache-report \
      --mm-attention-backend fa3 \
      --moe-runner-backend deep_gemm \
      --sampling-backend flashinfer \
      --moe-dense-tp-size 1 \
      --enable-dp-lm-head \
      --enable-dp-attention \
      --moe-a2a-backend deepep \
      --deepep-mode normal \
      --dist-init-addr $PREFILL_IP \
      --nnodes ${WORLD_SIZE} \
      --node-rank ${NODE_RANK} \
      --chunked-prefill-size -1 \
      --max-running-requests 64 \
      --disable-radix-cache \
      --port 20133 \
      --host 0.0.0.0 \
      --load-balance-method round_robin \
      --disaggregation-mode prefill \
      --disaggregation-ib-device $SGLANG_IB_DEVICES > "${LOG_DIR}/P${NODE_RANK}_${HOST_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

Decode启动脚本 decode_server.sh

#!/bin/bash
source ~/.virtualenvs/sglang-0.5.6/bin/activate

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export PATH=/root/.virtualenvs/sglang-0.5.6/bin/:$PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0

export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEP_GEMM_BLOCK_M=256
# 如果模型为 BF16 数据类型则打开该环境变量
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true

export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600

export SGLANG_DEEPEP_LL_USE_NVLINK=1
export SGLANG_DEEPEP_LL_DISABLE_RECV_HOOK=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128

export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1

# 获取当前环境的 RDMA 设备名列表，在启动时作为 disaggregation-ib-device 参数值
IB_DEVS=()
           
for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        # 速率（如 200 Gb/sec）
        rate=$(cat "$port/rate" 2>/dev/null)
        # link_layer: InfiniBand / Ethernet (RoCE)
        link=$(cat "$port/link_layer" 2>/dev/null)
        # 状态为 Up
        state=$(cat "$port/state" 2>/dev/null)
        # 只要是 200G/400G 且状态为 Up（IB 或 RoCE 都收）
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]] && [[ "$state" == "4"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
# 排序 + 去重
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
# 生成 MCCL / SGLang 变量
MCCL_IB_HCA=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
SGLANG_IB_DEVICES="$MCCL_IB_HCA"
echo "IB_DEVICES=$MCCL_IB_HCA"
# 获取 RDMA 设备名列表结束

NODE_RANK=$1
DECODER_IP="${2}:5403"
LOG_DIR=$3
MODEL_PATH=$4
HOST_IP=$5

WORLD_SIZE=1
export SGLANG_TORCH_PROFILER_DIR=$WORK_HOME/D${NODE_RANK}_traces

nohup python3 -m sglang.launch_server \
      --model-path ${MODEL_PATH} \
      --served-model-name qwen3-vl-235b-a22b-instruct-fp8 \
      --trust-remote-code \
      --disable-overlap-schedule \
      --cuda-graph-bs $(seq 1 32) \
      --tp-size 8 \
      --ep-size 8 \
      --dp-size 8 \
      --mem-fraction-static 0.76 \
      --attention-backend fa3 \
      --enable-cache-report \
      --mm-attention-backend fa3 \
      --moe-runner-backend deep_gemm \
      --enable-dp-lm-head \
      --moe-dense-tp-size 1 \
      --enable-dp-attention \
      --moe-a2a-backend deepep \
      --deepep-mode low_latency \
      --sampling-backend flashinfer \
      --dist-init-addr $DECODER_IP \
      --nnodes ${WORLD_SIZE} \
      --node-rank ${NODE_RANK} \
      --disable-radix-cache \
      --port 20143 \
      --host 0.0.0.0 \
      --ep-num-redundant-experts 0 \
      --load-balance-method round_robin \
      --prefill-round-robin-balance \
      --disaggregation-mode decode \
      --disaggregation-ib-device $SGLANG_IB_DEVICES > "${LOG_DIR}/D${NODE_RANK}_${HOST_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

Router启动脚本 router.sh

#!/bin/bash
source ~/.virtualenvs/sglang-0.5.6/bin/activate

export PATH=/root/.virtualenvs/sglang-0.5.6/bin/:$PATH
PREFILL_IP="http://${1}:20133"
DECODE_IP="http://${2}:20143"
LOG_DIR=$3

ulimit -n 65535
nohup python -m sglang_router.launch_router \
    --pd-disaggregation \
    --mini-lb \
    --prefill $PREFILL_IP \
    --decode $DECODE_IP \
    --host 0.0.0.0 \
    --request-timeout-secs 7200 \
    --port 30000 > "${LOG_DIR}/router_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

hostfile文件示例

192.168.100.101
192.168.100.102

一键运行脚本 run.sh

#!/bin/bash
WORKSPACE=$(pwd)

# 检查 hostfile 是否存在
if [ ! -f "$WORKSPACE/hostfile" ]; then
    echo "Error: $WORKSPACE/hostfile not found!"
    exit 1
fi

# 读取 hostfile 到数组
mapfile -t hosts < "$WORKSPACE/hostfile"

echo "Using hosts:"
printf '%s\n' "${hosts[@]}"

# prefill - 使用第一个节点作为 prefill 服务器
ssh -p 62216 ${hosts[0]} "cd $WORKSPACE && bash prefill_server.sh 0 ${hosts[0]} ./logs /data/models/Qwen3-VL-235B-A22B-Instruct-FP8/ ${hosts[0]}" &

# decode - 使用第二个节点作为 decoder 服务器
ssh -p 62216 ${hosts[1]} "cd $WORKSPACE && bash decode_server.sh 0 ${hosts[1]} ./logs /data/models/Qwen3-VL-235B-A22B-Instruct-FP8/ ${hosts[1]}" &

# router - 在第一个节点上启动 router
ssh -p 62216 ${hosts[0]} "cd $WORKSPACE && bash router.sh ${hosts[0]} ${hosts[1]} ./logs" &

echo "All services started in background"