跳到主要内容

Qwen3.5 模型部署

本章节包含 Qwen3.5 系列模型的部署脚本，脚本涉及的参数说明、PD 分离一键部署说明见 快速开始。

信息

启动 SGLang 服务前使用命令切换python虚拟环境到triton-3.2.0：workon triton-3.2.0或 source ~/.virtualenvs/triton-3.2.0/bin/activate

清理triton缓存：rm -rf ~/.triton/cache/ && rm -r /tmp/*

Qwen3.5-0.8B​

本示例使用 1卡（TP1） 部署。

启动脚本 qwen35_0.8b_server.sh

#!/bin/bash
rm -rf ~/.triton/cache/ && rm -r /tmp/*
source ~/.virtualenvs/triton-3.2.0/bin/activate
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export PATH=/root/.virtualenvs/triton-3.2.0/bin/:$PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1

get_master_ip() {
    local hostname_ip=$(hostname -I 2>/dev/null | awk '{print $1}')
  
    if [ -n "$hostname_ip" ] && [ "$hostname_ip" != "127.0.0.1" ]; then
        echo "$hostname_ip"
        return
    fi
  
}

CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
SGLANG_DECODE_PORT=20133
MASTER_IP=$(get_master_ip)
NODE_RANK=0
WORLD_SIZE=1

WORK_HOME="$PWD"
LOG_DIR=$WORK_HOME/output/$CURRENT_TIME/

mkdir -p "$LOG_DIR"
rm -rf ~/.triton/cache/ && rm -r /tmp/*
export MUSA_VISIBLE_DEVICES=3
export SGLANG_TORCH_PROFILER_DIR=$WORK_HOME/traces
MODEL_PATH=/data/models/Qwen3.5-0.8B/
nohup python3 -m sglang.launch_server \
      --model-path ${MODEL_PATH} \
      --served-model-name qwen3.5-0.8b \
      --trust-remote-code \
      --cuda-graph-bs $(seq 1 32) \
      --disable-cuda-graph-padding \
      --disable-overlap-schedule \
      --tp-size 1 \
      --pp-size 1 \
      --mem-fraction-static 0.8 \
      --attention-backend fa3 \
      --enable-cache-report \
      --mm-attention-backend fa3 \
      --moe-runner-backend deep_gemm \
      --sampling-backend flashinfer \
      --dist-init-addr ${MASTER_IP}:5303 \
      --nnodes ${WORLD_SIZE} \
      --node-rank ${NODE_RANK} \
      --chunked-prefill-size 4096 \
      --disable-radix-cache \
      --max-running-requests 32 \
      --max-prefill-tokens 8192 \
      --speculative-algo NEXTN \
      --speculative-num-steps 1 \
      --speculative-eagle-topk 1 \
      --speculative-num-draft-tokens 2 \
      --port ${SGLANG_DECODE_PORT} \
      --host 0.0.0.0 \
      --decode-log-interval 1 > "${LOG_DIR}/${MASTER_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

echo -e "Main log file: \033[34m$LOG_DIR\033[0m"

Qwen3.5-2B​

本示例使用 1卡（TP1） 部署。

启动脚本 qwen35_2b_server.sh

#!/bin/bash
rm -rf ~/.triton/cache/ && rm -r /tmp/*
source ~/.virtualenvs/triton-3.2.0/bin/activate
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export PATH=/root/.virtualenvs/triton-3.2.0/bin/:$PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1

get_master_ip() {
    local hostname_ip=$(hostname -I 2>/dev/null | awk '{print $1}')
  
    if [ -n "$hostname_ip" ] && [ "$hostname_ip" != "127.0.0.1" ]; then
        echo "$hostname_ip"
        return
    fi
  
}

CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
SGLANG_DECODE_PORT=20133
MASTER_IP=$(get_master_ip)
NODE_RANK=0
WORLD_SIZE=1

WORK_HOME="$PWD"
LOG_DIR=$WORK_HOME/output/$CURRENT_TIME/
mkdir -p "$LOG_DIR"

export MUSA_VISIBLE_DEVICES=2
export SGLANG_TORCH_PROFILER_DIR=$WORK_HOME/traces
MODEL_PATH=/data/models/Qwen3.5-2B/
nohup python3 -m sglang.launch_server \
      --model-path ${MODEL_PATH} \
      --served-model-name qwen3.5-2b \
      --trust-remote-code \
      --cuda-graph-bs $(seq 1 32) \
      --disable-cuda-graph-padding \
      --disable-overlap-schedule \
      --tp-size 1 \
      --pp-size 1 \
      --mem-fraction-static 0.8 \
      --attention-backend fa3 \
      --enable-cache-report \
      --mm-attention-backend fa3 \
      --moe-runner-backend deep_gemm \
      --sampling-backend flashinfer \
      --dist-init-addr ${MASTER_IP}:5303 \
      --nnodes ${WORLD_SIZE} \
      --node-rank ${NODE_RANK} \
      --chunked-prefill-size 4096 \
      --disable-radix-cache \
      --max-running-requests 32 \
      --max-prefill-tokens 8192 \
      --speculative-algo NEXTN \
      --speculative-num-steps 1 \
      --speculative-eagle-topk 1 \
      --speculative-num-draft-tokens 2 \
      --port ${SGLANG_DECODE_PORT} \
      --host 0.0.0.0 \
      --decode-log-interval 1 > "${LOG_DIR}/${MASTER_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

echo -e "Main log file: \033[34m$LOG_DIR\033[0m"

Qwen3.5-4B​

本示例使用 1卡（TP1） 部署。

启动脚本 qwen35_4b_server.sh

#!/bin/bash
rm -rf ~/.triton/cache/ && rm -r /tmp/*
source ~/.virtualenvs/triton-3.2.0/bin/activate

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export PATH=/root/.virtualenvs/triton-3.2.0/bin/:$PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1

get_master_ip() {
    local hostname_ip=$(hostname -I 2>/dev/null | awk '{print $1}')
  
    if [ -n "$hostname_ip" ] && [ "$hostname_ip" != "127.0.0.1" ]; then
        echo "$hostname_ip"
        return
    fi
  
}

CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
SGLANG_DECODE_PORT=20133
MASTER_IP=$(get_master_ip)
NODE_RANK=0
WORLD_SIZE=1

WORK_HOME="$PWD"
LOG_DIR=$WORK_HOME/output/$CURRENT_TIME/
mkdir -p "$LOG_DIR"

export MUSA_VISIBLE_DEVICES=1
export SGLANG_TORCH_PROFILER_DIR=$WORK_HOME/traces
MODEL_PATH=/data/models/Qwen3.5-4B/
nohup python3 -m sglang.launch_server \
      --model-path ${MODEL_PATH} \
      --served-model-name qwen3.5-4b \
      --trust-remote-code \
      --cuda-graph-bs $(seq 1 32) \
      --disable-cuda-graph-padding \
      --disable-overlap-schedule \
      --tp-size 1 \
      --pp-size 1 \
      --mem-fraction-static 0.8 \
      --attention-backend fa3 \
      --enable-cache-report \
      --mm-attention-backend fa3 \
      --moe-runner-backend deep_gemm \
      --sampling-backend flashinfer \
      --dist-init-addr ${MASTER_IP}:5303 \
      --nnodes ${WORLD_SIZE} \
      --node-rank ${NODE_RANK} \
      --chunked-prefill-size 4096 \
      --disable-radix-cache \
      --max-running-requests 32 \
      --max-prefill-tokens 8192 \
      --speculative-algo NEXTN \
      --speculative-num-steps 1 \
      --speculative-eagle-topk 1 \
      --speculative-num-draft-tokens 2 \
      --port ${SGLANG_DECODE_PORT} \
      --host 0.0.0.0 \
      --decode-log-interval 1 > "${LOG_DIR}/${MASTER_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

echo -e "Main log file: \033[34m$LOG_DIR\033[0m"

Qwen3.5-9B​

本示例使用 1卡（TP1） 部署。

启动脚本 qwen35_9b_server.sh

#!/bin/bash
rm -rf ~/.triton/cache/ && rm -r /tmp/*
source ~/.virtualenvs/triton-3.2.0/bin/activate

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export PATH=/root/.virtualenvs/triton-3.2.0/bin/:$PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1

get_master_ip() {
    local hostname_ip=$(hostname -I 2>/dev/null | awk '{print $1}')
  
    if [ -n "$hostname_ip" ] && [ "$hostname_ip" != "127.0.0.1" ]; then
        echo "$hostname_ip"
        return
    fi
  
}

CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
SGLANG_DECODE_PORT=20133
MASTER_IP=$(get_master_ip)
NODE_RANK=0
WORLD_SIZE=1

WORK_HOME="$PWD"
LOG_DIR=$WORK_HOME/output/$CURRENT_TIME/
mkdir -p "$LOG_DIR"

export MUSA_VISIBLE_DEVICES=0
export SGLANG_TORCH_PROFILER_DIR=$WORK_HOME/traces
MODEL_PATH=/data/models/Qwen3.5-9B/
nohup python3 -m sglang.launch_server \
      --model-path ${MODEL_PATH} \
      --served-model-name qwen3.5-9b \
      --trust-remote-code \
      --cuda-graph-bs $(seq 1 32) \
      --disable-cuda-graph-padding \
      --disable-overlap-schedule \
      --tp-size 1 \
      --pp-size 1 \
      --mem-fraction-static 0.8 \
      --attention-backend fa3 \
      --enable-cache-report \
      --mm-attention-backend fa3 \
      --moe-runner-backend deep_gemm \
      --sampling-backend flashinfer \
      --dist-init-addr ${MASTER_IP}:5303 \
      --nnodes ${WORLD_SIZE} \
      --node-rank ${NODE_RANK} \
      --chunked-prefill-size 4096 \
      --disable-radix-cache \
      --max-running-requests 32 \
      --max-prefill-tokens 8192 \
      --speculative-algo NEXTN \
      --speculative-num-steps 1 \
      --speculative-eagle-topk 1 \
      --speculative-num-draft-tokens 2 \
      --port ${SGLANG_DECODE_PORT} \
      --host 0.0.0.0 \
      --decode-log-interval 1 > "${LOG_DIR}/${MASTER_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

echo -e "Main log file: \033[34m$LOG_DIR\033[0m"

Qwen3.5-27B​

本示例使用 2卡（TP2） 部署， 也设置 --tp-size 1 进行单卡部署。

启动脚本 qwen35_27b_server.sh

#!/bin/bash
rm -rf ~/.triton/cache/ && rm -r /tmp/*
source ~/.virtualenvs/triton-3.2.0/bin/activate

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export PATH=/root/.virtualenvs/triton-3.2.0/bin/:$PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1

get_master_ip() {
    local hostname_ip=$(hostname -I 2>/dev/null | awk '{print $1}')
  
    if [ -n "$hostname_ip" ] && [ "$hostname_ip" != "127.0.0.1" ]; then
        echo "$hostname_ip"
        return
    fi
  
}

CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
SGLANG_DECODE_PORT=20133
MASTER_IP=$(get_master_ip)
NODE_RANK=0
WORLD_SIZE=1
WORK_HOME="$PWD"
LOG_DIR=$WORK_HOME/output/$CURRENT_TIME/
mkdir -p "$LOG_DIR"

export SGLANG_TORCH_PROFILER_DIR=$WORK_HOME/traces
MODEL_PATH=/data/models/Qwen3.5-27B-FP8/
nohup python3 -m sglang.launch_server \
      --model-path ${MODEL_PATH} \
      --served-model-name qwen3.5-27b-fp8 \
      --trust-remote-code \
      --cuda-graph-bs $(seq 1 32) \
      --disable-cuda-graph-padding \
      --disable-overlap-schedule \
      --tp-size 2 \
      --pp-size 1 \
      --mem-fraction-static 0.8 \
      --attention-backend fa3 \
      --enable-cache-report \
      --mm-attention-backend fa3 \
      --moe-runner-backend deep_gemm \
      --sampling-backend flashinfer \
      --dist-init-addr ${MASTER_IP}:5303 \
      --nnodes ${WORLD_SIZE} \
      --node-rank ${NODE_RANK} \
      --chunked-prefill-size 4096 \
      --disable-radix-cache \
      --max-running-requests 32 \
      --max-prefill-tokens 8192 \
      --context-length 16384 \
      --speculative-algo NEXTN \
      --speculative-num-steps 1 \
      --speculative-eagle-topk 1 \
      --speculative-num-draft-tokens 2 \
      --port ${SGLANG_DECODE_PORT} \
      --host 0.0.0.0 \
      --decode-log-interval 1 > "${LOG_DIR}/${MASTER_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

echo -e "Main log file: \033[34m$LOG_DIR\033[0m"

Qwen3.5-35B-A3B​

本示例使用 4卡（TP4） 部署。

启动脚本 qwen35_35b_a3b_server.sh

#!/bin/bash
rm -rf ~/.triton/cache/ && rm -r /tmp/*
source ~/.virtualenvs/triton-3.2.0/bin/activate

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export PATH=/root/.virtualenvs/triton-3.2.0/bin/:$PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1

get_master_ip() {
    local hostname_ip=$(hostname -I 2>/dev/null | awk '{print $1}')
  
    if [ -n "$hostname_ip" ] && [ "$hostname_ip" != "127.0.0.1" ]; then
        echo "$hostname_ip"
        return
    fi
  
}

CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
SGLANG_DECODE_PORT=20133
MASTER_IP=$(get_master_ip)
NODE_RANK=0
WORLD_SIZE=1
WORK_HOME="$PWD"
LOG_DIR=$WORK_HOME/output/$CURRENT_TIME/
mkdir -p "$LOG_DIR"

export SGLANG_TORCH_PROFILER_DIR=$WORK_HOME/traces
MODEL_PATH=/data/models/Qwen3.5-35B-A3B-FP8/
nohup python3 -m sglang.launch_server \
      --model-path ${MODEL_PATH} \
      --served-model-name qwen3.5-35b-a3b-fp8 \
      --trust-remote-code \
      --cuda-graph-bs $(seq 1 32) \
      --disable-overlap-schedule \
      --tp-size 4 \
      --pp-size 1 \
      --ep-size 4 \
      --mem-fraction-static 0.8 \
      --attention-backend fa3 \
      --enable-cache-report \
      --mm-attention-backend fa3 \
      --moe-runner-backend deep_gemm \
      --sampling-backend flashinfer \
      --dist-init-addr ${MASTER_IP}:5303 \
      --nnodes ${WORLD_SIZE} \
      --node-rank ${NODE_RANK} \
      --chunked-prefill-size -1 \
      --disable-radix-cache \
      --max-running-requests 32 \
      --max-prefill-tokens 8192 \
      --context-length 16384 \
      --speculative-algo NEXTN \
      --speculative-num-steps 1 \
      --speculative-eagle-topk 1 \
      --speculative-num-draft-tokens 2 \
      --port ${SGLANG_DECODE_PORT} \
      --host 0.0.0.0 \
      --decode-log-interval 1 > "${LOG_DIR}/${MASTER_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

echo -e "Main log file: \033[34m$LOG_DIR\033[0m"

Qwen3.5-122B-A10B​

本示例使用 1P1D 分离时部署，请准备2台机器。

Prefill启动脚本 prefill_server.sh

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export PATH=/root/.virtualenvs/sglang-0.5.6/bin/:$PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0

export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEP_GEMM_BLOCK_M=256
# 如果模型为 BF16 数据类型则打开该环境变量
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true

export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600

export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1

export SGLANG_EXPERT_DISTRIBUTION_RECORDER_DIR="$PWD"/Prefill-qwen3.5

NODE_RANK=$1
PREFILL_IP="${2}:5303"
LOG_DIR=$3
MODEL_PATH=$4
HOST_IP=$5

WORLD_SIZE=1
WORK_HOME="$PWD"
export SGLANG_TORCH_PROFILER_DIR=$WORK_HOME/P${NODE_RANK}_traces
export SGLANG_EXPERT_DISTRIBUTION_RECORDER_DIR=$WORK_HOME/Prefill-qwen3.5
DEEP_EP_CONFIG=$WORK_HOME/deepep.config

# 获取当前环境的 RDMA 设备名列表，在启动时作为 disaggregation-ib-device 参数值
IB_DEVS=()
         
for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        # 速率（如 200 Gb/sec）
        rate=$(cat "$port/rate" 2>/dev/null)
        # link_layer: InfiniBand / Ethernet (RoCE)
        link=$(cat "$port/link_layer" 2>/dev/null)
        # 状态为 Up
        state=$(cat "$port/state" 2>/dev/null)
        # 只要是 200G/400G 且状态为 Up（IB 或 RoCE 都收）
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]] && [[ "$state" == "4"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
# 排序 + 去重
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
# 生成 MCCL / SGLang 变量
MCCL_IB_HCA=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
SGLANG_IB_DEVICES="$MCCL_IB_HCA"
echo "IB_DEVICES=$MCCL_IB_HCA"
# 获取 RDMA 设备名列表结束

mkdir -p "${LOG_DIR}"

nohup python3 -m sglang.launch_server \
--model-path ${MODEL_PATH} \
      --served-model-name qwen3.5-122b-a10b \
      --trust-remote-code \
      --disable-overlap-schedule \
      --disable-cuda-graph \
      --tp-size 8 \
      --dp-size 8 \
      --ep-size 8 \
      --pp-size 1 \
      --mem-fraction-static 0.8 \
      --attention-backend fa3 \
      --enable-cache-report \
      --mm-attention-backend fa3 \
      --moe-runner-backend deep_gemm \
      --sampling-backend flashinfer \
      --moe-dense-tp-size 1 \
      --enable-dp-lm-head \
      --enable-dp-attention \
      --moe-a2a-backend deepep \
      --deepep-mode normal \
      --dist-init-addr $PREFILL_IP \
      --nnodes ${WORLD_SIZE} \
      --node-rank ${NODE_RANK} \
      --deepep-config "$DEEP_EP_CONFIG" \
      --chunked-prefill-size 4096 \
      --max-running-requests 64 \
      --disable-radix-cache \
      --port 20133 \
      --host 0.0.0.0 \
      --speculative-algo NEXTN \
      --speculative-num-steps 1 \
      --speculative-eagle-topk 1 \
      --speculative-num-draft-tokens 2 \
      --load-balance-method round_robin \
      --disaggregation-mode prefill \
      --disaggregation-ib-device $SGLANG_IB_DEVICES > "${LOG_DIR}/P${NODE_RANK}_${HOST_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

Decode启动脚本 decode_server.sh

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export PATH=/root/.virtualenvs/sglang-0.5.6/bin/:$PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0

export MC_ENABLE_DEST_DEVICE_AFFINITY=1
export MC_TE_METRIC=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEP_GEMM_BLOCK_M=256
# 如果模型为 BF16 数据类型则打开该环境变量
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true

export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600

export SGLANG_DEEPEP_LL_USE_NVLINK=1
export SGLANG_DEEPEP_LL_DISABLE_RECV_HOOK=0
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128

export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1

export SGLANG_DISAGGREGATION_MAPPING_IB_DEVICE_TO_GPU=1
export SGLANG_BLACKWELL_OVERLAP_SHARED_EXPERTS_OUTSIDE_SBO=0

WORK_HOME="$PWD"
export SGLANG_EXPERT_DISTRIBUTION_RECORDER_DIR=$WORK_HOME/Decode-qwen3.5

NODE_RANK=$1
DECODER_IP="${2}:5403"
LOG_DIR=$3
MODEL_PATH=$4
HOST_IP=$5

WORLD_SIZE=1
export SGLANG_TORCH_PROFILER_DIR=$WORK_HOME/D${NODE_RANK}_traces

# 获取当前环境的 RDMA 设备名列表，在启动时作为 disaggregation-ib-device 参数值
IB_DEVS=()
         
for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        # 速率（如 200 Gb/sec）
        rate=$(cat "$port/rate" 2>/dev/null)
        # link_layer: InfiniBand / Ethernet (RoCE)
        link=$(cat "$port/link_layer" 2>/dev/null)
        # 状态为 Up
        state=$(cat "$port/state" 2>/dev/null)
        # 只要是 200G/400G 且状态为 Up（IB 或 RoCE 都收）
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]] && [[ "$state" == "4"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
# 排序 + 去重
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
# 生成 MCCL / SGLang 变量
MCCL_IB_HCA=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
SGLANG_IB_DEVICES="$MCCL_IB_HCA"
echo "IB_DEVICES=$MCCL_IB_HCA"
# 获取 RDMA 设备名列表结束

nohup python3 -m sglang.launch_server \
      --model-path ${MODEL_PATH} \
      --trust-remote-code \
      --disable-overlap-schedule \
      --cuda-graph-bs $(seq 1 32) \
      --tp-size 8 \
      --ep-size 8 \
      --dp-size 8 \
      --mem-fraction-static 0.65 \
      --attention-backend fa3 \
      --enable-cache-report \
      --mm-attention-backend fa3 \
      --moe-runner-backend deep_gemm \
      --enable-dp-lm-head \
      --moe-dense-tp-size 1 \
      --enable-dp-attention \
      --moe-a2a-backend deepep \
      --deepep-mode low_latency \
      --sampling-backend flashinfer \
      --dist-init-addr $DECODER_IP \
      --nnodes ${WORLD_SIZE} \
      --node-rank ${NODE_RANK} \
      --disable-radix-cache \
      --port 20143 \
      --host 0.0.0.0 \
      --decode-log-interval 1 \
      --speculative-algo NEXTN \
      --speculative-num-steps 1 \
      --speculative-eagle-topk 1 \
      --speculative-num-draft-tokens 2 \
      --ep-num-redundant-experts 0 \
      --load-balance-method round_robin \
      --prefill-round-robin-balance \
      --disaggregation-mode decode \
      --disaggregation-ib-device $SGLANG_IB_DEVICES > "${LOG_DIR}/D${NODE_RANK}_${HOST_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

Router启动脚本 router.sh

export PATH=/root/.virtualenvs/triton-3.2.0/bin/:$PATH
PREFILL_IP="http://${1}:20133"
DECODE_IP="http://$2:20143"
LOG_DIR=$3

ulimit -n 65535
nohup python -m sglang_router.launch_router \
    --pd-disaggregation \
    --prefill $PREFILL_IP \
    --decode $DECODE_IP \
    --host 0.0.0.0 \
    --request-timeout-secs 7200 \
    --port 30000 > "${LOG_DIR}/router_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

hostfile文件示例

192.168.100.101
192.168.100.102

一键运行脚本 run.sh

#!/bin/bash

CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
#echo $CURRENT_TIME
#mkdir -p ./output/$CURRENT_TIME

PREFILL_SERVER_COUNT=${PREFILL_SERVER_COUNT:-1}
DECODER_SERVER_COUNT=${DECODER_SERVER_COUNT:-1}
WORLD_SIZE=$(( (PREFILL_SERVER_COUNT + DECODER_SERVER_COUNT) * 8 ))

echo -e "\033[32mPREFILL_SERVER_COUNT: $PREFILL_SERVER_COUNT, DECODER_SERVER_COUNT: $DECODER_SERVER_COUNT, \033[0m"

set -u
  WORK_HOME="$PWD"
  EXPNAME="P${PREFILL_SERVER_COUNT}_D${DECODER_SERVER_COUNT}_gpus${WORLD_SIZE}"
  MODEL_PATH=${MODEL_PATH:-"/data/models/Qwen3.5-122B-A10B-FP8/"}
  HOSTFILE=./hostfile
  LOG_DIR=$WORK_HOME/output/$EXPNAME/$CURRENT_TIME/
  PREFILL_SCRIPT_FILE="$WORK_HOME/prefill_server.sh"
  DECODER_SCRIPT_FILE="$WORK_HOME/decode_server.sh"
  ROUTER_SCRIPT_FILE="$WORK_HOME/router.sh"
  #ROUTER_SCRIPT_FILE="$WORK_HOME/router_sgl.sh"
set +u

echo -e "\033[32mWORK_HOME: $WORK_HOME\033[0m"
echo -e "\033[32mMODEL_PATH: $MODEL_PATH\033[0m"
echo -e "\033[32mPREFILL_CRIPT_FILE: $PREFILL_SCRIPT_FILE\033[0m"
echo -e "\033[32mDECODER_SCRIPT_FILE: $DECODER_SCRIPT_FILE\033[0m"

# bash generate_hostfile.sh 
COUNT=0
hostlist=$(grep -v '^#\|^$' $HOSTFILE | awk '{print $1}' | xargs)

# Check if hostlist is empty
if [ -z "$hostlist" ]; then
  echo "Error: hostlist is empty. Please add IP addresses to the hostfile."
  exit 1
fi

PREFILL_MASTER_IP=$(head -n 1 $HOSTFILE)
DECODER_MASTER_IP=$(tail -n $DECODER_SERVER_COUNT $HOSTFILE | head -n1)
echo -e "\033[32mPREFILL_MASTER_IP: $PREFILL_MASTER_IP\033[0m"
echo -e "\033[32mDECODER_MASTER_IP: $DECODER_MASTER_IP\033[0m"


mkdir -p $LOG_DIR
for host in ${hostlist[@]}; do
  echo $host
  if [[ $COUNT -lt $PREFILL_SERVER_COUNT ]]; then
    PREFILL_INDEX=$COUNT
    cmd="bash -c 'cd $WORK_HOME;source ~/.virtualenvs/triton-3.2.0/bin/activate; bash $PREFILL_SCRIPT_FILE $PREFILL_INDEX $PREFILL_MASTER_IP $LOG_DIR $MODEL_PATH $host'"
    #cmd="bash -c 'cd $WORK_HOME; echo $PATH '"
    echo $cmd
    ssh -p 62216 -f -n $host $cmd
  else
    DECODER_INDEX=$((COUNT - PREFILL_SERVER_COUNT))
    cmd="bash -c 'cd $WORK_HOME;source ~/.virtualenvs/triton-3.2.0/bin/activate; bash $DECODER_SCRIPT_FILE $DECODER_INDEX $DECODER_MASTER_IP $LOG_DIR $MODEL_PATH $host'"
    echo $cmd
    ssh -p 62216 -f -n $host $cmd
  fi

  ((COUNT++))
done
cmd="bash -c 'cd $WORK_HOME;source ~/.virtualenvs/triton-3.2.0/bin/activate; bash $ROUTER_SCRIPT_FILE $PREFILL_MASTER_IP $DECODER_MASTER_IP $LOG_DIR'"
echo $cmd
ssh -p 62216 -f -n $PREFILL_MASTER_IP $cmd
echo -e "Main log file: \033[34m$LOG_DIR\033[0m"

Qwen3.5-397B-A17B-FP8​

本示例使用 1P4D 分离式部署，请准备5台机器。DeepEP 的配置文件内容参考 DeepEP 配置文件

Prefill启动脚本 prefill_server.sh

#!/bin/bash
source ~/.virtualenvs/triton-3.2.0/bin/activate

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export PATH=/root/.virtualenvs/triton-3.2.0/bin/:$PATH

export TP_SOCKET_IFNAME=bond0
export GLOO_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1

export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_PROTOS=2
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0

export MC_TE_METRIC=1
export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_DEEP_GEMM_BLOCK_M=256
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true
export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_DISAGGREGATION_MAPPING_IB_DEVICE_TO_GPU=1

# 获取当前环境的 RDMA 设备名列表，在启动时作为 disaggregation-ib-device 参数值
IB_DEVS=()
         
for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        # 速率（如 200 Gb/sec）
        rate=$(cat "$port/rate" 2>/dev/null)
        # link_layer: InfiniBand / Ethernet (RoCE)
        link=$(cat "$port/link_layer" 2>/dev/null)
        # 状态为 Up
        state=$(cat "$port/state" 2>/dev/null)
        # 只要是 200G/400G 且状态为 Up（IB 或 RoCE 都收）
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]] && [[ "$state" == "4"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
# 排序 + 去重
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
# 生成 MCCL / SGLang 变量
MCCL_IB_HCA=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
SGLANG_IB_DEVICES="$MCCL_IB_HCA"
echo "IB_DEVICES=$MCCL_IB_HCA"
# 获取 RDMA 设备名列表结束

SGLANG_PORT=20133
MASTER_IP=$1
NODE_RANK=$2
WORLD_SIZE=1
DEEP_EP_CONFIG=$SCRIPT_DIR/deepep.config
MODEL_PATH=/data/models/Qwen3.5-397B-A17B-FP8

rm -rf ~/.triton/cache/ && rm -r /tmp/*

nohup python3 -m sglang.launch_server \
    --model $MODEL_PATH \
    --served-model-name qwen3.5-397b-a17b-fp8 \
    --trust-remote-code \
    --disable-cuda-graph \
    --disable-overlap-schedule \
    --tp-size 8 \
    --ep-size 8 \
    --dp-size 8 \
    --pp-size 1 \
    --moe-dense-tp-size 1 \
    --enable-dp-lm-head \
    --enable-dp-attention \
    --moe-a2a-backend deepep \
    --deepep-mode normal \
    --mem-fraction-static 0.8 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:5303 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size -1 \
    --disable-radix-cache \
    --enable-cache-report \
    --deepep-config "$DEEP_EP_CONFIG" \
    --schedule-conservativeness 1 \
    --max-running-requests 64 \
    --port ${SGLANG_PORT} \
    --host 0.0.0.0 \
    --speculative-algorithm NEXTN \
    --speculative-num-steps 1 \
    --speculative-eagle-topk 1 \
    --speculative-num-draft-tokens 2 \
    --load-balance-method round_robin \
    --disaggregation-mode prefill \
    --disaggregation-ib-device $SGLANG_IB_DEVICES > ./logs/prefill_server_$NODE_RANK.log 2>&1 &

Decode启动脚本 decode_server.sh

#!/bin/bash
source ~/.virtualenvs/triton-3.2.0/bin/activate

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export PATH=/root/.virtualenvs/triton-3.2.0/bin/:$PATH

export TP_SOCKET_IFNAME=bond0
export GLOO_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MATE_FORCE_JIT=1

export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_PROTOS=2
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0

export MC_TE_METRIC=1
export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_DEEPEP_LL_USE_NVLINK=1
export SGLANG_DEEPEP_LL_DISABLE_RECV_HOOK=0
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true
export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_DISAGGREGATION_MAPPING_IB_DEVICE_TO_GPU=1
export SGLANG_BLACKWELL_OVERLAP_SHARED_EXPERTS_OUTSIDE_SBO=0

# 获取当前环境的 RDMA 设备名列表，在启动时作为 disaggregation-ib-device 参数值
IB_DEVS=()
         
for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        # 速率（如 200 Gb/sec）
        rate=$(cat "$port/rate" 2>/dev/null)
        # link_layer: InfiniBand / Ethernet (RoCE)
        link=$(cat "$port/link_layer" 2>/dev/null)
        # 状态为 Up
        state=$(cat "$port/state" 2>/dev/null)
        # 只要是 200G/400G 且状态为 Up（IB 或 RoCE 都收）
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]] && [[ "$state" == "4"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
# 排序 + 去重
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
# 生成 MCCL / SGLang 变量
MCCL_IB_HCA=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
SGLANG_IB_DEVICES="$MCCL_IB_HCA"
echo "IB_DEVICES=$MCCL_IB_HCA"
# 获取 RDMA 设备名列表结束

SGLANG_PORT=20143
WORLD_SIZE=4
MASTER_IP=$1
NODE_RANK=$2
MODEL_PATH=/data/models/Qwen3.5-397B-A17B-FP8

rm -rf ~/.triton/cache/ && rm -r /tmp/*

nohup python3 -m sglang.launch_server \
    --model $MODEL_PATH \
    --served-model-name qwen3.5-397b-a17b-fp8 \
    --trust-remote-code \
    --cuda-graph-bs $(seq 1 32) \
    --disable-cuda-graph-padding \
    --tp-size $((WORLD_SIZE * 8)) \
    --ep-size $((WORLD_SIZE * 8)) \
    --dp-size $((WORLD_SIZE * 8)) \
    --enable-dp-lm-head \
    --moe-dense-tp-size 1 \
    --enable-dp-attention \
    --moe-a2a-backend deepep \
    --deepep-mode low_latency \
    --mem-fraction-static 0.65 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --max-running-requests 256 \
    --dist-init-addr ${MASTER_IP}:5403 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size -1 \
    --disable-radix-cache \
    --port ${SGLANG_PORT} \
    --host 0.0.0.0 \
    --decode-log-interval 1 \
    --prefill-round-robin-balance \
    --speculative-algorithm NEXTN \
    --speculative-num-steps 1 \
    --speculative-eagle-topk 1 \
    --speculative-num-draft-tokens 2 \
    --disaggregation-mode decode \
    --disaggregation-ib-device $SGLANG_IB_DEVICES > ./logs/decode_server_$NODE_RANK.log 2>&1 &

Router启动脚本 router.sh

#!/bin/bash
source ~/.virtualenvs/triton-3.2.0/bin/activate

export PATH=/root/.virtualenvs/triton-3.2.0/bin/:$PATH
PREFILL_IP="http://${1}:20133"
DECODE_IP="http://${2}:20143"
LOG_DIR=./logs

nohup python -m sglang_router.launch_router \
    --pd-disaggregation \
    --prefill $PREFILL_IP \
    --decode $DECODE_IP \
    --host 0.0.0.0 \
    --request-timeout-secs 7200 \
    --worker-startup-timeout-secs 600 \
    --port 30000 > "${LOG_DIR}/router.log" 2>&1 &

hostfile文件示例

192.168.100.101
192.168.100.102
192.168.100.103
192.168.100.104
192.168.100.105

一键运行脚本 run.sh

#!/bin/bash
WORKSPACE=$(pwd)

# 检查 hostfile 是否存在
if [ ! -f "$WORKSPACE/hostfile" ]; then
    echo "Error: $WORKSPACE/hostfile not found!"
    exit 1
fi

# 读取 hostfile 到数组
mapfile -t hosts < "$WORKSPACE/hostfile"

echo "Using hosts:"
printf '%s\n' "${hosts[@]}"

# prefill - 使用第一个节点作为 prefill 服务器
ssh -p 62216 ${hosts[0]} "source ~/.virtualenvs/triton-3.2.0/bin/activate && cd $WORKSPACE && bash prefill_server.sh ${hosts[0]} 0" &

# decode - 使用接下来的四个节点作为 decode 服务器
ssh -p 62216 ${hosts[1]} "source ~/.virtualenvs/triton-3.2.0/bin/activate && cd $WORKSPACE && bash decode_server.sh ${hosts[1]} 0" &
ssh -p 62216 ${hosts[2]} "source ~/.virtualenvs/triton-3.2.0/bin/activate && cd $WORKSPACE && bash decode_server.sh ${hosts[1]} 1" &
ssh -p 62216 ${hosts[3]} "source ~/.virtualenvs/triton-3.2.0/bin/activate && cd $WORKSPACE && bash decode_server.sh ${hosts[1]} 2" &
ssh -p 62216 ${hosts[4]} "source ~/.virtualenvs/triton-3.2.0/bin/activate && cd $WORKSPACE && bash decode_server.sh ${hosts[1]} 3" &

# router - 在第一个节点上启动 router
ssh -p 62216 ${hosts[0]} "source ~/.virtualenvs/triton-3.2.0/bin/activate && cd $WORKSPACE && bash router.sh ${hosts[0]} ${hosts[1]}" &

echo "All services started in background"

版权所有 2024 摩尔线程，本文档受国际版权法保护。

摩尔线程和摩尔线程徽标是摩尔线程智能科技 (北京)有限责任公司的注册商标。

免责声明

本文档提供有关摩尔线程产品的信息。本文档并未授权任何知识产权的许可，并未以明示或暗示，或以禁止反言或其他方式授予任何知识产权许可。