Qwen3 Dense 模型部署

本章节包含 Qwen3 Dense 系列模型的部署脚本，脚本涉及的参数说明、PD 分离一键部署说明见 快速开始。

info

启动 SGLang 服务前需要切换python虚拟环境到sglang-0.5.6，执行命令：workon sglang-0.5.6 或 source ~/.virtualenvs/sglang-0.5.6/bin/activate

Qwen3-8B

单卡启动脚本 qwen3_8b_server.sh

#!/bin/bash
source ~/.virtualenvs/sglang-0.5.6/bin/activate

export MCCL_SOCKET_IFNAME=bond0
export SGLANG_ENABLE_SPEC_V2=1
export MATE_FORCE_JIT=1
MODEL_PATH=/data/models/Qwen3-8B-FP8
SPEC_MODEL_PATH=/data/models/Qwen3-8B_eagle3

python3 -m sglang.launch_server \
            --model-path $MODEL_PATH \
            --served-model-name qwen3-8b-fp8 \
            --mem-fraction-static 0.9 \
            --cuda-graph-bs $(seq 1 64) \
            --host 0.0.0.0 \
            --port 20133 \
            --attention-backend fa3 \
            --tp-size 1 \
            --max-running-requests 256 \
            --sampling-backend flashinfer \
            --speculative-draft-model-path $SPEC_MODEL_PATH \
            --speculative-algorithm EAGLE3 \
            --speculative-num-steps 3 \
            --speculative-eagle-topk 1 \
            --speculative-num-draft-tokens 4 \
            --disable-radix-cache \
            --chunked-prefill-size 8192

Qwen3-14B

单卡启动脚本 qwen3_14b_server.sh

#!/bin/bash
source ~/.virtualenvs/sglang-0.5.6/bin/activate

export SGLANG_ENABLE_SPEC_V2=1
export MATE_FORCE_JIT=1
MODEL_PATH=/data/models/Qwen3-14B-FP8

python3 -m sglang.launch_server \
            --model-path $MODEL_PATH \
            --served-model-name qwen3-14b-fp8 \
            --mem-fraction-static 0.9 \
            --cuda-graph-bs $(seq 1 64) \
            --host 0.0.0.0 \
            --port 20133 \
            --attention-backend fa3 \
            --tp-size 1 \
            --max-running-requests 256 \
            --sampling-backend flashinfer \
            --disable-radix-cache \
            --chunked-prefill-size 8192

Qwen3-32B

Qwen3-32B 可以使用单机4卡混合部署，也可以在1台机器内做PD分离部署。

四卡 tp4 混合部署

Qwen3-32B tp4混部署启动脚本 qwen3_32b_tp4_server.sh

#!/bin/bash
source ~/.virtualenvs/sglang-0.5.6/bin/activate

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export SGLANG_TORCH_PROFILER_DIR=/tmp/traces
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1
export LD_LIBRARY_PATH=/usr/local/musa/lib:${LD_LIBRARY_PATH}

SGLANG_DECODE_PORT=20133
MASTER_IP=127.0.0.1
NODE_RANK=0
WORLD_SIZE=1
MODEL_PATH=/data/models/Qwen3-32B/
log_path=logs-sglang-server/$(hostname)
mkdir -p $log_path
log_file=$(date "+%Y%m%d_%H%M").log
touch $log_path/$log_file
rm -f $log_path/latest
ln -s $log_file $log_path/latest
python3 -m sglang.launch_server \
    --model ${MODEL_PATH} \
    --served-model-name qwen3-32b \
    --trust-remote-code \
    --cuda-graph-max-bs 256 \
    --disable-overlap-schedule \
    --tp-size 4 \
    --dp-size 1 \
    --mem-fraction-static 0.8 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:5303 \
    --nnodes 1 \
    --node-rank 0 \
    --chunked-prefill-size -1 \
    --disable-radix-cache \
    --port ${SGLANG_DECODE_PORT} \
    --host 0.0.0.0 \
    --speculative-algorithm EAGLE3 \
    --speculative-num-steps 1 \
    --speculative-eagle-topk 1 \
    --speculative-num-draft-tokens 2 \
    --speculative-draft-model-path /data/models/Qwen3-32B_eagle3/ \
    --decode-log-interval 1 2>&1 | tee $log_path/$log_file

单机PD分离部署

该示例使用1台机器，4张卡做Prefill，4张卡做Decoed。

Prefill启动脚本 prefill_server.sh

#!/bin/bash
source ~/.virtualenvs/sglang-0.5.6/bin/activate

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0

export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEP_GEMM_BLOCK_M=256
# 如果模型为 BF16 数据类型则打开该环境变量
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true

export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600

export SGLANG_TORCH_PROFILER_DIR=/tmp/traces
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1

export SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True
export MC_FORCE_MNNVL=True

# 获取当前环境的 RDMA 设备名列表，在启动时作为 disaggregation-ib-device 参数值
IB_DEVS=()
       
for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        # 速率（如 200 Gb/sec）
        rate=$(cat "$port/rate" 2>/dev/null)
        # link_layer: InfiniBand / Ethernet (RoCE)
        link=$(cat "$port/link_layer" 2>/dev/null)
        # 状态为 Up
        state=$(cat "$port/state" 2>/dev/null)
        # 只要是 200G/400G 且状态为 Up（IB 或 RoCE 都收）
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]] && [[ "$state" == "4"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
# 排序 + 去重
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
# 生成 MCCL / SGLang 变量
MCCL_IB_HCA=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
SGLANG_IB_DEVICES="$MCCL_IB_HCA"
echo "IB_DEVICES=$MCCL_IB_HCA"
# 获取 RDMA 设备名列表结束

SGLANG_PREFILL_PORT=20133
SGLANG_DECODE_PORT=20143
SGLANG_ROUTER_PORT=30000

MASTER_IP=127.0.0.1
NODE_RANK=0
WORLD_SIZE=1
MODEL_PATH=/data/models/Qwen3-32B

log_path=logs-sglang-server/$(hostname)
mkdir -p $log_path
times_now=$(date "+%Y%m%d_%H%M")

prefill_log_file=${times_now}_prefill.log
decode_log_file=${times_now}_decode.log
router_log_file=${times_now}_router.log

export MUSA_VISIBLE_DEVICES=0,1,2,3
nohup python3 -m sglang.launch_server \
    --model ${MODEL_PATH} \
    --served-model-name qwen3-32b \
    --trust-remote-code \
    --disable-cuda-graph \
    --disable-overlap-schedule \
    --tp-size 4 \
    --dp-size 1 \
    --mem-fraction-static 0.85 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:5303 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size 8192 \
    --disable-radix-cache \
    --enable-cache-report \
    --schedule-conservativeness 1 \
    --max-running-requests 64 \
    --max-prefill-tokens 4096 \
    --port ${SGLANG_PREFILL_PORT} \
    --host 0.0.0.0 \
    --load-balance-method round_robin \
    --disaggregation-mode prefill \
    --speculative-algorithm EAGLE3 \
    --speculative-num-steps 1 \
    --speculative-eagle-topk 1 \
    --speculative-num-draft-tokens 2 \
    --speculative-draft-model-path /data/models/Qwen3-32B_eagle3/ \
    --disaggregation-ib-device $SGLANG_IB_DEVICES > ${log_path}/${prefill_log_file} 2>&1 &
sleep 10

Decode启动脚本 decode_server.sh

#!/bin/bash
source ~/.virtualenvs/sglang-0.5.6/bin/activate

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0

export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEP_GEMM_BLOCK_M=256
# 如果模型为 BF16 数据类型则打开该环境变量
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true

export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600

export SGLANG_TORCH_PROFILER_DIR=/tmp/traces
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1

export SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True
export MC_FORCE_MNNVL=True

export SGLANG_DEEPEP_LL_USE_NVLINK=1
export SGLANG_DEEPEP_LL_DISABLE_RECV_HOOK=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128

# 获取当前环境的 RDMA 设备名列表，在启动时作为 disaggregation-ib-device 参数值
IB_DEVS=()
       
for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        # 速率（如 200 Gb/sec）
        rate=$(cat "$port/rate" 2>/dev/null)
        # link_layer: InfiniBand / Ethernet (RoCE)
        link=$(cat "$port/link_layer" 2>/dev/null)
        # 状态为 Up
        state=$(cat "$port/state" 2>/dev/null)
        # 只要是 200G/400G 且状态为 Up（IB 或 RoCE 都收）
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]] && [[ "$state" == "4"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
# 排序 + 去重
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
# 生成 MCCL / SGLang 变量
MCCL_IB_HCA=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
SGLANG_IB_DEVICES="$MCCL_IB_HCA"
echo "IB_DEVICES=$MCCL_IB_HCA"
# 获取 RDMA 设备名列表结束

SGLANG_PREFILL_PORT=20133
SGLANG_DECODE_PORT=20143
SGLANG_ROUTER_PORT=30000

MASTER_IP=127.0.0.1
NODE_RANK=0
WORLD_SIZE=1
MODEL_PATH=/data/models/Qwen3-32B

log_path=logs-sglang-server/$(hostname)
mkdir -p $log_path
times_now=$(date "+%Y%m%d_%H%M")

prefill_log_file=${times_now}_prefill.log
decode_log_file=${times_now}_decode.log
router_log_file=${times_now}_router.log

export MUSA_VISIBLE_DEVICES=4,5,6,7

nohup python3 -m sglang.launch_server \
    --model ${MODEL_PATH} \
    --trust-remote-code \
    --cuda-graph-max-bs 256  \
    --tp-size 4 \
    --dp-size 1 \
    --mem-fraction-static 0.8 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --max-running-requests 256 \
    --dist-init-addr ${MASTER_IP}:5403 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size -1 \
    --disable-radix-cache \
    --port ${SGLANG_DECODE_PORT} \
    --host 0.0.0.0 \
    --decode-log-interval 1 \
    --load-balance-method round_robin \
    --prefill-round-robin-balance \
    --disaggregation-mode decode \
    --speculative-algorithm EAGLE3 \
    --speculative-num-steps 1 \
    --speculative-eagle-topk 1 \
    --speculative-num-draft-tokens 2 \
    --speculative-draft-model-path /data/models/Qwen3-32B_eagle3/ \
    --disaggregation-ib-device $SGLANG_IB_DEVICES > ${log_path}/${decode_log_file} 2>&1 &

Router启动脚本 router.sh

#!/bin/bash
source ~/.virtualenvs/sglang-0.5.6/bin/activate

SGLANG_PREFILL_PORT=20133
SGLANG_DECODE_PORT=20143
SGLANG_ROUTER_PORT=30000

PREFILL_IP="http://127.0.0.1:$SGLANG_PREFILL_PORT"
DECODE_IP="http://127.0.0.1:$SGLANG_DECODE_PORT"

log_path=logs-sglang-server/$(hostname)
mkdir -p $log_path
times_now=$(date "+%Y%m%d_%H%M")
router_log_file=${times_now}_router.log

nohup python -m sglang_router.launch_router \
    --pd-disaggregation \
    --mini-lb \
    --prefill $PREFILL_IP \
    --decode $DECODE_IP \
    --host 0.0.0.0 \
    --request-timeout-secs 7200 \
    --port $SGLANG_ROUTER_PORT > ${log_path}/${router_log_file} 2>&1 &