跳到主要内容

Qwen3 MoE 模型部署

本章节包含 Qwen3 MOE 系列模型的部署脚本，脚本涉及的参数说明、PD 分离一键部署说明见 快速开始。

信息

启动 SGLang 服务前需要切换python虚拟环境到sglang-0.5.6 workon sglang-0.5.6 或 source ~/.virtualenvs/sglang-0.5.6/bin/activate

Qwen3-Coder-480B-A35B-Instruct​

本示例使用 1P2D 分离部署，如需更佳性能，建议使用 2P4D 部署方式。最小部署方式为2机混布（如 TP8 PP2）。

注意，如果是BF16版本需要在prefill_server.sh 以及decode_server.sh添加/修改环境变量: export SGLANG_DEEPEP_BF16_DISPATCH=1

Prefill启动脚本 prefill_server.sh

#!/bin/bash
source ~/.virtualenvs/sglang-0.5.6/bin/activate
WORKSPACE=$(pwd)
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

export TP_SOCKET_IFNAME=bond0
export GLOO_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1

export MUSA_LAUNCH_BLOCKING=0

export MCCL_PROTOS=2
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0

export MC_TE_METRIC=1
export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_DEEP_GEMM_BLOCK_M=256
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true
export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_DISAGGREGATION_MAPPING_IB_DEVICE_TO_GPU=1

export SGLANG_TORCH_PROFILER_DIR=$WORKSPACE/traces
export SGLANG_EXPERT_DISTRIBUTION_RECORDER_DIR=$WORKSPACE/traces
export VLLM_CONFIGURE_LOGGING=0
export MC_TE_METRIC=true

export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1

# 获取当前环境的 RDMA 设备名列表，在启动时作为 disaggregation-ib-device 参数值
IB_DEVS=()
   
for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        # 速率（如 200 Gb/sec）
        rate=$(cat "$port/rate" 2>/dev/null)
        # link_layer: InfiniBand / Ethernet (RoCE)
        link=$(cat "$port/link_layer" 2>/dev/null)
        # 状态为 Up
        state=$(cat "$port/state" 2>/dev/null)
        # 只要是 200G/400G 且状态为 Up（IB 或 RoCE 都收）
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]] && [[ "$state" == "4"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
# 排序 + 去重
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
# 生成 MCCL / SGLang 变量
MCCL_IB_HCA=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
SGLANG_IB_DEVICES="$MCCL_IB_HCA"
echo "IB_DEVICES=$MCCL_IB_HCA"
# 获取 RDMA 设备名列表结束

SGLANG_PORT=20133
MASTER_IP=$1
NODE_RANK=$2
WORLD_SIZE=1
DEEP_EP_CONFIG=$WORKSPACE/deepep.config
MODEL_PATH=/data/models/Qwen3-Coder-480B-A35B-Instruct-FP8/

mkdir -p ./logs
# pip install transformers==4.57.1
nohup /root/.virtualenvs/sglang-0.5.6/bin/python3 -m sglang.launch_server \
    --model $MODEL_PATH \
    --served-model-name qwen3-coder-480b-a35b-instruct-fp8 \
    --trust-remote-code \
    --disable-cuda-graph \
    --disable-overlap-schedule \
    --tp-size 8 \
    --ep-size 8 \
    --dp-size 8 \
    --pp-size 1 \
    --enable-dp-lm-head \
    --enable-dp-attention \
    --moe-dense-tp-size 1 \
    --moe-a2a-backend deepep \
    --moe-runner-backend deep_gemm \
    --deepep-mode normal \
    --mem-fraction-static 0.90 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:5303 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size 32768 \
    --disable-radix-cache \
    --enable-cache-report \
    --deepep-config "$DEEP_EP_CONFIG" \
    --schedule-conservativeness 1 \
    --max-running-requests 64 \
    --port ${SGLANG_PORT} \
    --host 0.0.0.0 \
    --load-balance-method round_robin \
    --disaggregation-mode prefill \
    --disaggregation-ib-device $SGLANG_IB_DEVICES > ./logs/coder_prefill_server_$NODE_RANK.log 2>&1 &

Decode启动脚本 decode_server.sh

#!/bin/bash
source ~/.virtualenvs/sglang-0.5.6/bin/activate
WORKSPACE=$(pwd)
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

export TP_SOCKET_IFNAME=bond0
export GLOO_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1
# export PYTORCH_MUSA_ALLOC_CONF=expandable_segments:True

export MUSA_LAUNCH_BLOCKING=0

export MCCL_PROTOS=2
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0

export MC_TE_METRIC=1
export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_DEEPEP_LL_USE_NVLINK=1
export SGLANG_DEEPEP_LL_DISABLE_RECV_HOOK=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true
export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_DISAGGREGATION_MAPPING_IB_DEVICE_TO_GPU=1
export SGLANG_TORCH_PROFILER_DIR=$WORKSPACE/traces
export SGLANG_EXPERT_DISTRIBUTION_RECORDER_DIR=$WORKSPACE/traces
export VLLM_CONFIGURE_LOGGING=0
export MC_TE_METRIC=true

# 获取当前环境的 RDMA 设备名列表，在启动时作为 disaggregation-ib-device 参数值
IB_DEVS=()
   
for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        # 速率（如 200 Gb/sec）
        rate=$(cat "$port/rate" 2>/dev/null)
        # link_layer: InfiniBand / Ethernet (RoCE)
        link=$(cat "$port/link_layer" 2>/dev/null)
        # 状态为 Up
        state=$(cat "$port/state" 2>/dev/null)
        # 只要是 200G/400G 且状态为 Up（IB 或 RoCE 都收）
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]] && [[ "$state" == "4"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
# 排序 + 去重
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
# 生成 MCCL / SGLang 变量
MCCL_IB_HCA=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
SGLANG_IB_DEVICES="$MCCL_IB_HCA"
echo "IB_DEVICES=$MCCL_IB_HCA"
# 获取 RDMA 设备名列表结束


SGLANG_PORT=20143
WORLD_SIZE=2
MASTER_IP=$1
NODE_RANK=$2
MODEL_PATH=/data/models/Qwen3-Coder-480B-A35B-Instruct-FP8/

mkdir -p ./logs
# pip install transformers==4.57.1
nohup /root/.virtualenvs/sglang-0.5.6/bin/python3 -m sglang.launch_server \
    --model $MODEL_PATH \
    --served-model-name qwen3-coder-480b-a35b-instruct-fp8 \
    --trust-remote-code \
    --cuda-graph-max-bs 64  \
    --tp-size $((WORLD_SIZE * 8)) \
    --ep-size $((WORLD_SIZE * 8)) \
    --dp-size $((WORLD_SIZE * 8)) \
    --enable-dp-lm-head \
    --enable-dp-attention \
    --moe-dense-tp-size 1 \
    --moe-a2a-backend deepep \
    --deepep-mode low_latency \
    --mem-fraction-static 0.92 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --moe-runner-backend deep_gemm \
    --max-running-requests 512 \
    --dist-init-addr ${MASTER_IP}:5403 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size -1 \
    --disable-radix-cache \
    --port ${SGLANG_PORT} \
    --host 0.0.0.0 \
    --decode-log-interval 1 \
    --prefill-round-robin-balance \
    --disaggregation-mode decode \
    --disaggregation-ib-device $SGLANG_IB_DEVICES > ./logs/coder_decode_server_$NODE_RANK.log 2>&1 &

Router启动脚本 router.sh

source ~/.virtualenvs/sglang-0.5.6/bin/activate
PREFILL_IP="http://${1}:20133"
DECODE_IP="http://${2}:20143"
LOG_DIR=./logs


nohup /root/.virtualenvs/sglang-0.5.6/bin/python3 -m sglang_router.launch_router \
    --pd-disaggregation \
    --prefill $PREFILL_IP \
    --decode $DECODE_IP \
    --host 0.0.0.0 \
    --request-timeout-secs 7200 \
    --worker-startup-timeout-secs 600 \
    --port 30000 > "${LOG_DIR}/coder_router.log" 2>&1 &

一键启动脚本 run.sh

WORKSPACE=$(pwd)

# 检查 hostfile 是否存在
if [ ! -f "$WORKSPACE/hostfile" ]; then
    echo "Error: $WORKSPACE/hostfile not found!"
    exit 1
fi

# 读取 hostfile 到数组
mapfile -t hosts < "$WORKSPACE/hostfile"

# 检查是否读取到足够的节点
if [ ${#hosts[@]} -lt 3 ]; then
    echo "Error: hostfile should contain at least 3 nodes"
    exit 1
fi

echo "Using hosts:"
printf '%s\n' "${hosts[@]}"

# prefill - 使用第一个节点作为 prefill 服务器
ssh -p 62216 ${hosts[0]} "cd $WORKSPACE && bash prefill_server.sh ${hosts[0]} 0" &
# ssh ${hosts[1]} "cd $WORKSPACE && bash prefill_server.sh ${hosts[0]} 1" &
sleep 2
# decode - 使用接下来的两个节点作为 decode 服务器
ssh -p 62216 ${hosts[1]} "cd $WORKSPACE && bash decode_server.sh ${hosts[1]} 0" &
sleep 2
ssh -p 62216 ${hosts[2]} "cd $WORKSPACE && bash decode_server.sh ${hosts[1]} 1" &
sleep 2
# router - 在第一个节点上启动 router
ssh -p 62216 ${hosts[0]} "cd $WORKSPACE && bash router.sh ${hosts[0]} ${hosts[1]}" &

echo "All services started in background"

Qwen3-235B-A22B-Instruct-2507​

本示例使用 1P2D 分离部署，如需更佳性能，建议使用 2P4D 部署方式。 最小部署方式为2机混布（如 TP8 PP2）。

注意，如果是BF16版本需要在prefill_server.sh 以及decode_server.sh添加/修改环境变量。

export SGLANG_DEEPEP_BF16_DISPATCH=1

Prefill启动脚本 prefill_server.sh

source ~/.virtualenvs/sglang-0.5.6/bin/activate
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

export TP_SOCKET_IFNAME=bond0
export GLOO_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1
# export PYTORCH_MUSA_ALLOC_CONF=expandable_segments:True

export MUSA_LAUNCH_BLOCKING=0

export MCCL_PROTOS=2
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0

export MC_TE_METRIC=1
export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_DEEP_GEMM_BLOCK_M=256
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true
export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_DISAGGREGATION_MAPPING_IB_DEVICE_TO_GPU=1

export VLLM_CONFIGURE_LOGGING=0
export MC_TE_METRIC=true

# 获取当前环境的 RDMA 设备名列表，在启动时作为 disaggregation-ib-device 参数值
IB_DEVS=()
   
for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        # 速率（如 200 Gb/sec）
        rate=$(cat "$port/rate" 2>/dev/null)
        # link_layer: InfiniBand / Ethernet (RoCE)
        link=$(cat "$port/link_layer" 2>/dev/null)
        # 状态为 Up
        state=$(cat "$port/state" 2>/dev/null)
        # 只要是 200G/400G 且状态为 Up（IB 或 RoCE 都收）
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]] && [[ "$state" == "4"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
# 排序 + 去重
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
# 生成 MCCL / SGLang 变量
MCCL_IB_HCA=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
SGLANG_IB_DEVICES="$MCCL_IB_HCA"
echo "IB_DEVICES=$MCCL_IB_HCA"
# 获取 RDMA 设备名列表结束


SGLANG_PORT=20133
MASTER_IP=$1
NODE_RANK=$2
WORLD_SIZE=1
DEEP_EP_CONFIG=$SCRIPT_DIR/deepep.config
MODEL_PATH=/data/models/Qwen3-235B-A22B-Instruct-2507-FP8/

mkdir -p ./logs
# pip install transformers==4.57.1
nohup /root/.virtualenvs/sglang-0.5.6/bin/python3 -m sglang.launch_server \
    --model $MODEL_PATH \
    --served-model-name qwen3-235b-a22b-fp8 \
    --trust-remote-code \
    --disable-cuda-graph \
    --disable-overlap-schedule \
    --tp-size 8 \
    --ep-size 8 \
    --dp-size 8 \
    --pp-size 1 \
    --enable-dp-lm-head \
    --enable-dp-attention \
    --moe-dense-tp-size 1 \
    --moe-a2a-backend deepep \
    --moe-runner-backend deep_gemm \
    --deepep-mode normal \
    --mem-fraction-static 0.90 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:5303 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size 32768 \
    --disable-radix-cache \
    --enable-cache-report \
    --deepep-config "$DEEP_EP_CONFIG" \
    --schedule-conservativeness 1 \
    --max-running-requests 64 \
    --port ${SGLANG_PORT} \
    --host 0.0.0.0 \
    --speculative-draft-model /data/models/EAGLE3-Qwen3-235B-A22B-Instruct-2507-FP8/ \
    --speculative-algorithm EAGLE3 \
    --speculative-num-steps 1 \
    --speculative-eagle-topk 1 \
    --speculative-num-draft-tokens 2 \
    --load-balance-method round_robin \
    --disaggregation-mode prefill \
    --disaggregation-ib-device $SGLANG_IB_DEVICES  > ./logs/prefill_server_$NODE_RANK.log 2>&1 &

Decode启动脚本 decode_server.sh

#!/bin/bash
source ~/.virtualenvs/sglang-0.5.6/bin/activate
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

export TP_SOCKET_IFNAME=bond0
export GLOO_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1
# export PYTORCH_MUSA_ALLOC_CONF=expandable_segments:True

export MUSA_LAUNCH_BLOCKING=0

export MCCL_PROTOS=2
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0

export MC_TE_METRIC=1
export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_DEEPEP_LL_USE_NVLINK=1
export SGLANG_DEEPEP_LL_DISABLE_RECV_HOOK=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true
export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_DISAGGREGATION_MAPPING_IB_DEVICE_TO_GPU=1
export SGLANG_TORCH_PROFILER_DIR=$SCRIPT_DIR/traces
export SGLANG_EXPERT_DISTRIBUTION_RECORDER_DIR=$SCRIPT_DIR/traces
export VLLM_CONFIGURE_LOGGING=0
export MC_TE_METRIC=true

# 获取当前环境的 RDMA 设备名列表，在启动时作为 disaggregation-ib-device 参数值
IB_DEVS=()
   
for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        # 速率（如 200 Gb/sec）
        rate=$(cat "$port/rate" 2>/dev/null)
        # link_layer: InfiniBand / Ethernet (RoCE)
        link=$(cat "$port/link_layer" 2>/dev/null)
        # 状态为 Up
        state=$(cat "$port/state" 2>/dev/null)
        # 只要是 200G/400G 且状态为 Up（IB 或 RoCE 都收）
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]] && [[ "$state" == "4"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
# 排序 + 去重
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
# 生成 MCCL / SGLang 变量
MCCL_IB_HCA=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
SGLANG_IB_DEVICES="$MCCL_IB_HCA"
echo "IB_DEVICES=$MCCL_IB_HCA"
# 获取 RDMA 设备名列表结束


SGLANG_PORT=20143
WORLD_SIZE=2
MASTER_IP=$1
NODE_RANK=$2

MODEL_PATH=/data/models/Qwen3-235B-A22B-Instruct-2507-FP8/

mkdir -p ./logs

# pip install transformers==4.57.1
nohup /root/.virtualenvs/sglang-0.5.6/bin/python3 -m sglang.launch_server \
    --model $MODEL_PATH \
    --trust-remote-code \
    --cuda-graph-max-bs 64  \
    --tp-size $((WORLD_SIZE * 8)) \
    --ep-size $((WORLD_SIZE * 8)) \
    --dp-size $((WORLD_SIZE * 8)) \
    --enable-dp-lm-head \
    --enable-dp-attention \
    --moe-dense-tp-size 1 \
    --moe-a2a-backend deepep \
    --deepep-mode low_latency \
    --mem-fraction-static 0.88 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --moe-runner-backend deep_gemm \
    --max-running-requests 512 \
    --dist-init-addr ${MASTER_IP}:5403 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size -1 \
    --disable-radix-cache \
    --port ${SGLANG_PORT} \
    --host 0.0.0.0 \
    --speculative-draft-model /data/models/EAGLE3-Qwen3-235B-A22B-Instruct-2507-FP8/ \
    --speculative-algorithm EAGLE3 \
    --speculative-num-steps 1 \
    --speculative-eagle-topk 1 \
    --speculative-num-draft-tokens 2 \
    --decode-log-interval 1 \
    --prefill-round-robin-balance \
    --disaggregation-mode decode \
    --disaggregation-ib-device $SGLANG_IB_DEVICES  > ./logs/decode_server_$NODE_RANK.log 2>&1 &

Router启动脚本 router.sh

source ~/.virtualenvs/sglang-0.5.6/bin/activate
PREFILL_IP="http://${1}:20133"
DECODE_IP="http://$2:20143"
LOG_DIR=./logs

nohup /root/.virtualenvs/sglang-0.5.6/bin/python3 -m sglang_router.launch_router \
    --pd-disaggregation \
    --prefill $PREFILL_IP \
    --decode $DECODE_IP \
    --host 0.0.0.0 \
    --mini-lb \
    --request-timeout-secs 7200 \
    --worker-startup-timeout-secs 600 \
    --port 30000 > "${LOG_DIR}/coder_router.log" 2>&1 &

一键运行脚本 run.sh

WORKSPACE=$(pwd)

# 检查 hostfile 是否存在
if [ ! -f "$WORKSPACE/hostfile" ]; then
    echo "Error: $WORKSPACE/hostfile not found!"
    exit 1
fi

# 读取 hostfile 到数组
mapfile -t hosts < "$WORKSPACE/hostfile"

# 检查是否读取到足够的节点
if [ ${#hosts[@]} -lt 3 ]; then
    echo "Error: hostfile should contain at least 3 nodes"
    exit 1
fi

echo "Using hosts:"
printf '%s\n' "${hosts[@]}"

# prefill - 使用第一个节点作为 prefill 服务器
ssh -p 62216 ${hosts[0]} "cd $WORKSPACE && bash prefill_server.sh ${hosts[0]} 0" &
# ssh ${hosts[1]} "cd $WORKSPACE && bash prefill_server.sh ${hosts[0]} 1" &
sleep 3
# decode - 使用接下来的两个节点作为 decode 服务器
ssh -p 62216 ${hosts[1]} "cd $WORKSPACE && bash decode_server.sh ${hosts[1]} 0" &
sleep 5
ssh -p 62216 ${hosts[2]} "cd $WORKSPACE && bash decode_server.sh ${hosts[1]} 1" &
sleep 2
# router - 在第一个节点上启动 router
ssh -p 62216 ${hosts[0]} "cd $WORKSPACE && bash router.sh ${hosts[0]} ${hosts[1]}" &

echo "All services started in background"

Qwen3-30B-A3B​

本示例使用 单机4卡（TP4） 部署。

启动脚本 launch_server.sh

source ~/.virtualenvs/sglang-0.5.6/bin/activate
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export SGLANG_TORCH_PROFILER_DIR=/tmp/traces
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1
export LD_LIBRARY_PATH=/usr/local/musa/lib:${LD_LIBRARY_PATH}

SGLANG_DECODE_PORT=20133
MASTER_IP=127.0.0.1
NODE_RANK=0
WORLD_SIZE=1
MODEL_PATH=/data/models/Qwen3-30B-A3B/
log_path=logs-sglang-server/$(hostname)
mkdir -p $log_path
log_file=$(date "+%Y%m%d_%H%M").log
touch $log_path/$log_file
rm -f $log_path/latest
ln -s $log_file $log_path/latest
python3 -m sglang.launch_server \
    --model ${MODEL_PATH} \
    --trust-remote-code \
    --cuda-graph-max-bs 256 \
    --disable-overlap-schedule \
    --tp-size 4 \
    --ep-size 4 \
    --mem-fraction-static 0.8 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --moe-runner-backend deep_gemm \
    --sampling-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:5303 \
    --nnodes 1 \
    --node-rank 0 \
    --chunked-prefill-size -1 \
    --disable-radix-cache \
    --port ${SGLANG_DECODE_PORT} \
    --host 0.0.0.0 \
    --decode-log-interval 1 2>&1 | tee $log_path/$log_file
  #  logs-sglang-server/$(date "+%Y%m%d_%H%M").log

Qwen3-Next-80B-A3B-Instruct​

本示例使单机PD分离部署，Prefill 和 Decode 各使用4张卡（TP4）。

启动脚本 launch_server.sh

#!/bin/bash
source ~/.virtualenvs/sglang-0.5.6/bin/activate
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0

export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEP_GEMM_BLOCK_M=256
# 如果模型为 BF16 数据类型则打开该环境变量
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true

export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600

export SGLANG_TORCH_PROFILER_DIR=/tmp/traces
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1
export DEEP_EP_CONFIG=$SCRIPT_DIR/deepep.config

# 获取当前环境的 RDMA 设备名列表，在启动时作为 disaggregation-ib-device 参数值
IB_DEVS=()
   
for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        # 速率（如 200 Gb/sec）
        rate=$(cat "$port/rate" 2>/dev/null)
        # link_layer: InfiniBand / Ethernet (RoCE)
        link=$(cat "$port/link_layer" 2>/dev/null)
        # 状态为 Up
        state=$(cat "$port/state" 2>/dev/null)
        # 只要是 200G/400G 且状态为 Up（IB 或 RoCE 都收）
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]] && [[ "$state" == "4"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
# 排序 + 去重
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
# 生成 MCCL / SGLang 变量
MCCL_IB_HCA=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
SGLANG_IB_DEVICES="$MCCL_IB_HCA"
echo "IB_DEVICES=$MCCL_IB_HCA"
# 获取 RDMA 设备名列表结束



SGLANG_PREFILL_PORT=20133
SGLANG_DECODE_PORT=20143
SGLANG_ROUTER_PORT=30000


MASTER_IP=127.0.0.1
NODE_RANK=0
WORLD_SIZE=1
MODEL_PATH=/data/models/Qwen3-Next-80B-A3B-Instruct-FP8

log_path=logs-sglang-server/$(hostname)
mkdir -p $log_path
times_now=$(date "+%Y%m%d_%H%M")

prefill_log_file=${times_now}_prefill.log
decode_log_file=${times_now}_decode.log
router_log_file=${times_now}_reouter.log


export MUSA_VISIBLE_DEVICES=0,1,2,3
nohup python3 -m sglang.launch_server \
    --model ${MODEL_PATH} \
    --trust-remote-code \
    --disable-cuda-graph \
    --disable-overlap-schedule \
    --deepep-config "$DEEP_EP_CONFIG" \
    --tp-size 4 \
    --ep-size 4 \
    --dp-size 4 \
    --enable-dp-lm-head \
    --enable-dp-attention \
    --moe-dense-tp-size 1 \
    --moe-a2a-backend deepep \
    --deepep-mode normal \
    --mem-fraction-static 0.85 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:5303 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size -1 \
    --disable-radix-cache \
    --enable-cache-report \
    --schedule-conservativeness 1 \
    --max-running-requests 64 \
    --max-prefill-tokens 4096 \
    --ep-num-redundant-experts 0 \
    --port ${SGLANG_PREFILL_PORT} \
    --host 0.0.0.0 \
    --load-balance-method round_robin \
    --disaggregation-mode prefill \
    --disaggregation-ib-device $SGLANG_IB_DEVICES  > ${log_path}/${prefill_log_file} 2>&1 &
sleep 10

export SGLANG_DEEPEP_LL_USE_NVLINK=1
export SGLANG_DEEPEP_LL_DISABLE_RECV_HOOK=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128

export MUSA_VISIBLE_DEVICES=4,5,6,7

nohup python3 -m sglang.launch_server \
    --model ${MODEL_PATH} \
    --trust-remote-code \
    --cuda-graph-max-bs 16  \
    --tp-size 4 \
    --ep-size 4 \
    --dp-size 4 \
    --enable-dp-lm-head \
    --enable-dp-attention \
    --moe-dense-tp-size 1 \
    --moe-a2a-backend deepep \
    --deepep-mode low_latency \
    --mem-fraction-static 0.8 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --max-running-requests 16 \
    --dist-init-addr ${MASTER_IP}:5403 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --ep-num-redundant-experts 0 \
    --chunked-prefill-size -1 \
    --disable-radix-cache \
    --port ${SGLANG_DECODE_PORT} \
    --host 0.0.0.0 \
    --decode-log-interval 1 \
    --load-balance-method round_robin \
    --prefill-round-robin-balance \
    --disaggregation-mode decode \
    --disaggregation-ib-device $SGLANG_IB_DEVICES > ${log_path}/${decode_log_file} 2>&1 &



PREFILL_IP="http://127.0.0.1:$SGLANG_PREFILL_PORT"
DECODE_IP="http://127.0.0.1:$SGLANG_DECODE_PORT"

nohup python -m sglang_router.launch_router \
    --pd-disaggregation \
    --mini-lb \
    --prefill $PREFILL_IP \
    --decode $DECODE_IP \
    --host 0.0.0.0 \
    --request-timeout-secs 7200 \
    --port $SGLANG_ROUTER_PORT > ${log_path}/${router_log_file} 2>&1 &

while [ 0 -eq 0 ];do
	grep "The server is fired up and ready to roll!" ${log_path}/${decode_log_file} > /dev/null 2>&1
	if [ $? -eq 0 ];then
		break
	fi
done
echo "All logs are stored in ./logs"

版权所有 2024 摩尔线程，本文档受国际版权法保护。

摩尔线程和摩尔线程徽标是摩尔线程智能科技 (北京)有限责任公司的注册商标。

免责声明

本文档提供有关摩尔线程产品的信息。本文档并未授权任何知识产权的许可，并未以明示或暗示，或以禁止反言或其他方式授予任何知识产权许可。