跳到主要内容

Qwen3 MoE 模型部署

本章节包含 Qwen3 MOE 系列模型的部署脚本及说明。支持 BF16 和 FP8 两种权重，部署时注意替换权重文件。

  • 参考 环境准备 进行环境检查、启动容器。

  • 容器镜像地址参考 版本发布信息-镜像地址。

  • PD 分离一键部署说明见 快速开始。

  • 参考 环境准备-RDMA 网卡检测 中的脚本，请其保存为 resolve_sglang_pd_ib_devices.sh，与 SGLang 服务启动脚本放在同一路径下。

  • 部分脚本用到的 EPLB_FILE 是利用测试数据场景进行录制，可联系支持工程师获取。实际生产部署请自行录制，也可删除 EPLB_FILE 环境变量及 SGLang 启动参数中的 init-expert-location。

Qwen3-Coder-480B-A35B-Instruct​

模型链接（Qwen3-Coder-480B-A35B-Instruct）

提示

  • FP8模型可以采用1P2D方式部署，BF16模型需要采用2P4D方式部署。

  • 如果是BF16版本需要在prefill_server.sh 以及decode_server.sh添加/修改环境变量 export SGLANG_DEEPEP_BF16_DISPATCH=1

启动方法：将Prefill启动脚本，Decode启动脚本，Router启动脚本，run_sglang.sh和resolve_sglang_pd_ib_devices.sh放在同一路径下。并将需要使用的节点IP放进hostfile文件中。运行bash run_sglang.sh。

1P2D 分离部署​

一键运行脚本 run.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"
SSH_PORT="${SSH_PORT:-62216}"
WORKSPACE=$(pwd)
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="$WORK_HOME/output/$CURRENT_TIME/"
PREFILL_SERVER_COUNT=${PREFILL_SERVER_COUNT:-1}
DECODER_SERVER_COUNT=${DECODER_SERVER_COUNT:-2}
PREFILL_NNODES=${PREFILL_NNODES:-$PREFILL_SERVER_COUNT}
DECODER_NNODES=${DECODER_NNODES:-$DECODER_SERVER_COUNT}
REQUIRED_NODES=$((PREFILL_SERVER_COUNT + DECODER_SERVER_COUNT))
MODEL_PATH=/data/models/Qwen3-Coder-480B-A35B-Instruct-FP8/
mkdir -p "$LOG_DIR"

# 检查 hostfile 是否存在
if [ ! -f "$WORKSPACE/hostfile" ]; then
    echo "Error: $WORKSPACE/hostfile not found!"
    exit 1
fi

# 读取 hostfile 到数组
mapfile -t hosts < "$WORKSPACE/hostfile"

# 检查是否读取到足够的节点
if [[ ${#hosts[@]} -lt "$REQUIRED_NODES" ]]; then
    echo "Error: hostfile should contain at least $REQUIRED_NODES nodes"
    exit 1
fi

echo "Using hosts:"
printf '%s\n' "${hosts[@]}"
echo "MODEL_PATH: $MODEL_PATH"

PREFILL_MASTER_IP=${hosts[0]}
DECODER_MASTER_IP=${hosts[$PREFILL_SERVER_COUNT]}

# prefill
for ((prefill_rank = 0; prefill_rank < PREFILL_SERVER_COUNT; prefill_rank++)); do
    prefill_host=${hosts[$prefill_rank]}
    ssh -p "$SSH_PORT" "$prefill_host" "cd $WORKSPACE && SGLANG_VENV=$SGLANG_VENV SGLANG_PYTHON=$SGLANG_PYTHON bash prefill_server.sh $PREFILL_MASTER_IP $prefill_rank $PREFILL_NNODES $LOG_DIR $MODEL_PATH" &
done
sleep 2

# decode
for ((decode_rank = 0; decode_rank < DECODER_SERVER_COUNT; decode_rank++)); do
    decode_host_index=$((PREFILL_SERVER_COUNT + decode_rank))
    decode_host=${hosts[$decode_host_index]}
    ssh -p "$SSH_PORT" "$decode_host" "cd $WORKSPACE && SGLANG_VENV=$SGLANG_VENV SGLANG_PYTHON=$SGLANG_PYTHON bash decode_server.sh $DECODER_MASTER_IP $decode_rank $DECODER_NNODES $LOG_DIR $MODEL_PATH" &
    sleep 2
done

# router - 在第一个节点上启动 router
ssh -p "$SSH_PORT" "$PREFILL_MASTER_IP" "cd $WORKSPACE && SGLANG_VENV=$SGLANG_VENV SGLANG_PYTHON=$SGLANG_PYTHON bash router.sh $PREFILL_MASTER_IP $DECODER_MASTER_IP $LOG_DIR" &

echo "All services started in background"
echo "All logs are stored in $LOG_DIR"

prefill启动脚本 prefill_server.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

export TP_SOCKET_IFNAME=bond0
export GLOO_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
# export PYTORCH_MUSA_ALLOC_CONF=expandable_segments:True

export MUSA_LAUNCH_BLOCKING=0

export MCCL_PROTOS=2
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0

export MC_TE_METRIC=1
export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_DEEP_GEMM_BLOCK_M=256
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true
export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SGLANG_IB_TARGET_COUNT=8
SGLANG_IB_REPEAT_PER_DEVICE=2
SGLANG_PD_IB_RESOLVER="${SCRIPT_DIR}/resolve_sglang_pd_ib_devices.sh"
if [[ -f "$SGLANG_PD_IB_RESOLVER" ]]; then
    source "$SGLANG_PD_IB_RESOLVER"
else
    SGLANG_DISAGGREGATION_IB_DEVICES="${SGLANG_DISAGGREGATION_IB_DEVICES:-${SGLANG_IB_DEFAULT_DEVICES:-}}"
    export SGLANG_DISAGGREGATION_IB_DEVICES
    echo "Warning: ${SGLANG_PD_IB_RESOLVER} not found, use default IB devices"
    echo "IB_DEVICES=$SGLANG_DISAGGREGATION_IB_DEVICES"
fi

export VLLM_CONFIGURE_LOGGING=0
export MC_TE_METRIC=true

# export NVSHMEM_IB_TRAFFIC_CLASS=163
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
# export MUSA_DEVICE_PAGE_SIZE=0x1000
# export MUSA_MANAGED_FORCE_DEVICE_ALLOC=1
# export NVSHMEM_IBGDA_NIC_HANDLER=cpu

sleep 3

SGLANG_PORT="${SGLANG_PORT:-30133}"
MASTER_IP=$1
NODE_RANK=$2
NNODES=$3
WORLD_SIZE=$NNODES
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="${4:-$WORK_HOME/output/$CURRENT_TIME/}"
export SGLANG_TORCH_PROFILER_DIR="$WORK_HOME/traces/P${NODE_RANK}"

MODEL_PATH="${5:?MODEL_PATH must be passed from run_sglang.sh}"
mkdir -p "$LOG_DIR"
SHARED_DIR="${SGLANG_SHARED_DIR:-${SCRIPT_DIR}/shared}"
if [[ -z "${SGLANG_SHARED_DIR:-}" && ! -d "$SHARED_DIR" ]]; then
    for shared_candidate in "${SCRIPT_DIR}/../shared" "${SCRIPT_DIR}/../../shared" "${SCRIPT_DIR}/../../../shared" "${SCRIPT_DIR}/../../../../shared"; do
        if [[ -d "$shared_candidate" ]]; then
            SHARED_DIR="$(cd "$shared_candidate" && pwd)"
            break
        fi
    done
fi

DEEP_EP_CONFIG="/data/workspace/deepep/deepep.config"
echo "DEEP_EP_CONFIG=$DEEP_EP_CONFIG"
# pip install transformers==4.57.1
nohup "$SGLANG_PYTHON" -m sglang.launch_server \
    --disable-piecewise-cuda-graph \
    --model $MODEL_PATH \
    --trust-remote-code \
    --disable-cuda-graph \
    --disable-overlap-schedule \
    --tp-size 8 \
    --ep-size 8 \
    --dp-size 8 \
    --pp-size 1 \
    --enable-dp-lm-head \
    --enable-dp-attention \
    --moe-dense-tp-size 1 \
    --moe-a2a-backend deepep \
    --moe-runner-backend auto \
    --deepep-mode normal \
    --mem-fraction-static 0.90 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --tokenizer-backend fastokens \
    --linear-attn-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:5303 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size 32768 \
    --enable-cache-report \
    --deepep-config "$DEEP_EP_CONFIG" \
    --schedule-conservativeness 1 \
    --max-running-requests 64 \
    --port ${SGLANG_PORT} \
    --host "${SGLANG_HOST}" \
    --load-balance-method round_robin \
    --disaggregation-mode prefill \
    --disaggregation-ib-device "$SGLANG_DISAGGREGATION_IB_DEVICES" \
    --tool-call-parser qwen3_coder \
    > "${LOG_DIR}/coder_prefill_server_${NODE_RANK}.log" 2>&1 &

decoder启动脚本 decoder_server.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

export TP_SOCKET_IFNAME=bond0
export GLOO_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MUSA_LAUNCH_BLOCKING=0

export MCCL_PROTOS=2
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0

export MC_TE_METRIC=1
export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_DEEPEP_LL_USE_NVLINK=1
export SGLANG_DEEPEP_LL_DISABLE_RECV_HOOK=0
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true
export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_DISAGGREGATION_MAPPING_IB_DEVICE_TO_GPU=1

export VLLM_CONFIGURE_LOGGING=0
export MC_TE_METRIC=true

# export NVSHMEM_IB_TRAFFIC_CLASS=136
# export NVSHMEM_IB_TRAFFIC_CLASS=163
# export MUSA_DEVICE_PAGE_SIZE=0x1000
# export MUSA_MANAGED_FORCE_DEVICE_ALLOC=1
# export NVSHMEM_IBGDA_NIC_HANDLER=cpu
# export MUSA_EXECUTION_TIMEOUT=15000

#export SGLANG_ENABLE_SPEC_V2=1
#export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
sleep 3

SGLANG_PORT="${SGLANG_PORT:-30133}"
MASTER_IP=$1
NODE_RANK=$2
NNODES=$3
WORLD_SIZE=$NNODES
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="${4:-$WORK_HOME/output/$CURRENT_TIME/}"
export SGLANG_TORCH_PROFILER_DIR="$WORK_HOME/traces/D${NODE_RANK}"
MODEL_PATH="${5:?MODEL_PATH must be passed from run_sglang.sh}"
mkdir -p "$LOG_DIR"

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SGLANG_IB_TARGET_COUNT=8
SGLANG_IB_REPEAT_PER_DEVICE=2
SGLANG_PD_IB_RESOLVER="${SCRIPT_DIR}/resolve_sglang_pd_ib_devices.sh"
if [[ -f "$SGLANG_PD_IB_RESOLVER" ]]; then
    source "$SGLANG_PD_IB_RESOLVER"
else
    SGLANG_DISAGGREGATION_IB_DEVICES="${SGLANG_DISAGGREGATION_IB_DEVICES:-${SGLANG_IB_DEFAULT_DEVICES:-}}"
    export SGLANG_DISAGGREGATION_IB_DEVICES
    echo "Warning: ${SGLANG_PD_IB_RESOLVER} not found, use default IB devices"
    echo "IB_DEVICES=$SGLANG_DISAGGREGATION_IB_DEVICES"
fi
nohup "$SGLANG_PYTHON" -m sglang.launch_server \
    --disable-piecewise-cuda-graph \
    --model $MODEL_PATH \
    --trust-remote-code \
    --disable-overlap-schedule \
    --cuda-graph-max-bs 54  \
    --tp-size $((WORLD_SIZE * 8)) \
    --ep-size $((WORLD_SIZE * 8)) \
    --dp-size $((WORLD_SIZE * 8)) \
    --enable-dp-lm-head \
    --enable-dp-attention \
    --moe-dense-tp-size 1 \
    --moe-a2a-backend deepep \
    --deepep-mode low_latency \
    --mem-fraction-static 0.86 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --tokenizer-backend fastokens \
    --linear-attn-backend flashinfer \
    --moe-runner-backend auto \
    --max-running-requests 512 \
    --dist-init-addr ${MASTER_IP}:5302 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size -1 \
    --port ${SGLANG_PORT} \
    --host "${SGLANG_HOST}" \
    --decode-log-interval 1 \
    --prefill-round-robin-balance \
    --disaggregation-mode decode \
    --disaggregation-ib-device "$SGLANG_DISAGGREGATION_IB_DEVICES" \
    --tool-call-parser qwen3_coder \
    > "${LOG_DIR}/coder_decode_server_${NODE_RANK}.log" 2>&1 &

Router 启动脚本 router.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"
SGLANG_PREFILL_PORT="${SGLANG_PREFILL_PORT:-30133}"
SGLANG_DECODE_PORT="${SGLANG_DECODE_PORT:-30133}"
SGLANG_ROUTER_PORT="${SGLANG_ROUTER_PORT:-31000}"
PREFILL_IP="http://${1}:$SGLANG_PREFILL_PORT"
DECODE_IP="http://${2}:$SGLANG_DECODE_PORT"
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="${3:-$WORK_HOME/output/$CURRENT_TIME/}"
mkdir -p "$LOG_DIR"

lsof -ti:"$SGLANG_ROUTER_PORT" | xargs -r kill -9


nohup "$SGLANG_PYTHON" -m sglang_router.launch_router \
    --pd-disaggregation \
    --prefill "$PREFILL_IP" \
    --decode "$DECODE_IP" \
    --host "${SGLANG_HOST}" \
    --mini-lb \
    --request-timeout-secs 7200 \
    --worker-startup-timeout-secs 600 \
    --port "$SGLANG_ROUTER_PORT" > "${LOG_DIR}/coder_router.log" 2>&1 &

2P4D部署​

BF16模型

一键运行脚本 run.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"
SSH_PORT="${SSH_PORT:-62216}"
WORKSPACE=$(pwd)
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="$WORK_HOME/output/$CURRENT_TIME/"
PREFILL_SERVER_COUNT=${PREFILL_SERVER_COUNT:-2}
DECODER_SERVER_COUNT=${DECODER_SERVER_COUNT:-4}
PREFILL_NNODES=${PREFILL_NNODES:-$PREFILL_SERVER_COUNT}
DECODER_NNODES=${DECODER_NNODES:-$DECODER_SERVER_COUNT}
REQUIRED_NODES=$((PREFILL_SERVER_COUNT + DECODER_SERVER_COUNT))
MODEL_PATH=/data/models/Qwen3-Coder-480B-A35B-Instruct-FP8/
mkdir -p "$LOG_DIR"

# 检查 hostfile 是否存在
if [ ! -f "$WORKSPACE/hostfile" ]; then
    echo "Error: $WORKSPACE/hostfile not found!"
    exit 1
fi

# 读取 hostfile 到数组
mapfile -t hosts < "$WORKSPACE/hostfile"

# 检查是否读取到足够的节点
if [[ ${#hosts[@]} -lt "$REQUIRED_NODES" ]]; then
    echo "Error: hostfile should contain at least $REQUIRED_NODES nodes"
    exit 1
fi

echo "Using hosts:"
printf '%s\n' "${hosts[@]}"
echo "MODEL_PATH: $MODEL_PATH"

PREFILL_MASTER_IP=${hosts[0]}
DECODER_MASTER_IP=${hosts[$PREFILL_SERVER_COUNT]}

# prefill
for ((prefill_rank = 0; prefill_rank < PREFILL_SERVER_COUNT; prefill_rank++)); do
    prefill_host=${hosts[$prefill_rank]}
    ssh -p "$SSH_PORT" "$prefill_host" "cd $WORKSPACE && SGLANG_VENV=$SGLANG_VENV SGLANG_PYTHON=$SGLANG_PYTHON bash prefill_server.sh $PREFILL_MASTER_IP $prefill_rank $PREFILL_NNODES $LOG_DIR $MODEL_PATH" &
done
sleep 2

# decode
for ((decode_rank = 0; decode_rank < DECODER_SERVER_COUNT; decode_rank++)); do
    decode_host_index=$((PREFILL_SERVER_COUNT + decode_rank))
    decode_host=${hosts[$decode_host_index]}
    ssh -p "$SSH_PORT" "$decode_host" "cd $WORKSPACE && SGLANG_VENV=$SGLANG_VENV SGLANG_PYTHON=$SGLANG_PYTHON bash decode_server.sh $DECODER_MASTER_IP $decode_rank $DECODER_NNODES $LOG_DIR $MODEL_PATH" &
    sleep 2
done

# router - 在第一个节点上启动 router
ssh -p "$SSH_PORT" "$PREFILL_MASTER_IP" "cd $WORKSPACE && SGLANG_VENV=$SGLANG_VENV SGLANG_PYTHON=$SGLANG_PYTHON bash router.sh $PREFILL_MASTER_IP $DECODER_MASTER_IP $LOG_DIR" &

echo "All services started in background"
echo "All logs are stored in $LOG_DIR"

prefill启动脚本 prefill_server.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

export TP_SOCKET_IFNAME=bond0
export GLOO_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
# export PYTORCH_MUSA_ALLOC_CONF=expandable_segments:True

export MUSA_LAUNCH_BLOCKING=0

export MCCL_PROTOS=2
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0

export MC_TE_METRIC=1
export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_DEEP_GEMM_BLOCK_M=256
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEPEP_BF16_DISPATCH=1
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true
export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SGLANG_IB_TARGET_COUNT=8
SGLANG_IB_REPEAT_PER_DEVICE=2
SGLANG_PD_IB_RESOLVER="${SCRIPT_DIR}/resolve_sglang_pd_ib_devices.sh"
if [[ -f "$SGLANG_PD_IB_RESOLVER" ]]; then
    source "$SGLANG_PD_IB_RESOLVER"
else
    SGLANG_DISAGGREGATION_IB_DEVICES="${SGLANG_DISAGGREGATION_IB_DEVICES:-${SGLANG_IB_DEFAULT_DEVICES:-}}"
    export SGLANG_DISAGGREGATION_IB_DEVICES
    echo "Warning: ${SGLANG_PD_IB_RESOLVER} not found, use default IB devices"
    echo "IB_DEVICES=$SGLANG_DISAGGREGATION_IB_DEVICES"
fi

export VLLM_CONFIGURE_LOGGING=0
export MC_TE_METRIC=true

# export NVSHMEM_IB_TRAFFIC_CLASS=163
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
# export MUSA_DEVICE_PAGE_SIZE=0x1000
# export MUSA_MANAGED_FORCE_DEVICE_ALLOC=1
# export NVSHMEM_IBGDA_NIC_HANDLER=cpu

sleep 3

SGLANG_PORT="${SGLANG_PORT:-30133}"
MASTER_IP=$1
NODE_RANK=$2
NNODES=$3
WORLD_SIZE=$NNODES
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="${4:-$WORK_HOME/output/$CURRENT_TIME/}"
export SGLANG_TORCH_PROFILER_DIR="$WORK_HOME/traces/P${NODE_RANK}"

MODEL_PATH="${5:?MODEL_PATH must be passed from run_sglang.sh}"
mkdir -p "$LOG_DIR"
SHARED_DIR="${SGLANG_SHARED_DIR:-${SCRIPT_DIR}/shared}"
if [[ -z "${SGLANG_SHARED_DIR:-}" && ! -d "$SHARED_DIR" ]]; then
    for shared_candidate in "${SCRIPT_DIR}/../shared" "${SCRIPT_DIR}/../../shared" "${SCRIPT_DIR}/../../../shared" "${SCRIPT_DIR}/../../../../shared"; do
        if [[ -d "$shared_candidate" ]]; then
            SHARED_DIR="$(cd "$shared_candidate" && pwd)"
            break
        fi
    done
fi

DEEP_EP_CONFIG="/data/workspace/deepep/deepep.config"
echo "DEEP_EP_CONFIG=$DEEP_EP_CONFIG"
# pip install transformers==4.57.1
nohup "$SGLANG_PYTHON" -m sglang.launch_server \
    --disable-piecewise-cuda-graph \
    --model $MODEL_PATH \
    --trust-remote-code \
    --disable-cuda-graph \
    --disable-overlap-schedule \
    --tp-size 8 \
    --ep-size 8 \
    --dp-size 8 \
    --pp-size 2 \
    --enable-dp-lm-head \
    --enable-dp-attention \
    --moe-dense-tp-size 1 \
    --moe-a2a-backend deepep \
    --moe-runner-backend auto \
    --deepep-mode normal \
    --mem-fraction-static 0.90 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --tokenizer-backend fastokens \
    --linear-attn-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:5303 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size -1 \
    --enable-cache-report \
    --deepep-config "$DEEP_EP_CONFIG" \
    --schedule-conservativeness 1 \
    --max-running-requests 64 \
    --port ${SGLANG_PORT} \
    --host "${SGLANG_HOST}" \
    --load-balance-method round_robin \
    --disaggregation-mode prefill \
    --disaggregation-ib-device "$SGLANG_DISAGGREGATION_IB_DEVICES" \
    --tool-call-parser qwen3_coder \
    > "${LOG_DIR}/coder_prefill_server_${NODE_RANK}.log" 2>&1 &

decoder启动脚本 decoder_server.sh

export MUSA_LAUNCH_BLOCKING=0

export MCCL_PROTOS=2
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0

export MC_TE_METRIC=1
export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEPEP_BF16_DISPATCH=1
export SGLANG_DEEPEP_LL_USE_NVLINK=1
export SGLANG_DEEPEP_LL_DISABLE_RECV_HOOK=0
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true
export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_DISAGGREGATION_MAPPING_IB_DEVICE_TO_GPU=1

export VLLM_CONFIGURE_LOGGING=0
export MC_TE_METRIC=true

# export NVSHMEM_IB_TRAFFIC_CLASS=136
# export NVSHMEM_IB_TRAFFIC_CLASS=163
# export MUSA_DEVICE_PAGE_SIZE=0x1000
# export MUSA_MANAGED_FORCE_DEVICE_ALLOC=1
# export NVSHMEM_IBGDA_NIC_HANDLER=cpu
# export MUSA_EXECUTION_TIMEOUT=15000

#export SGLANG_ENABLE_SPEC_V2=1
#export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
sleep 3

SGLANG_PORT="${SGLANG_PORT:-30133}"
MASTER_IP=$1
NODE_RANK=$2
NNODES=$3
WORLD_SIZE=$NNODES
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="${4:-$WORK_HOME/output/$CURRENT_TIME/}"
export SGLANG_TORCH_PROFILER_DIR="$WORK_HOME/traces/D${NODE_RANK}"
MODEL_PATH="${5:?MODEL_PATH must be passed from run_sglang.sh}"
mkdir -p "$LOG_DIR"

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SGLANG_IB_TARGET_COUNT=8
SGLANG_IB_REPEAT_PER_DEVICE=2
SGLANG_PD_IB_RESOLVER="${SCRIPT_DIR}/resolve_sglang_pd_ib_devices.sh"
if [[ -f "$SGLANG_PD_IB_RESOLVER" ]]; then
    source "$SGLANG_PD_IB_RESOLVER"
else
    SGLANG_DISAGGREGATION_IB_DEVICES="${SGLANG_DISAGGREGATION_IB_DEVICES:-${SGLANG_IB_DEFAULT_DEVICES:-}}"
    export SGLANG_DISAGGREGATION_IB_DEVICES
    echo "Warning: ${SGLANG_PD_IB_RESOLVER} not found, use default IB devices"
    echo "IB_DEVICES=$SGLANG_DISAGGREGATION_IB_DEVICES"
fi
nohup "$SGLANG_PYTHON" -m sglang.launch_server \
    --disable-piecewise-cuda-graph \
    --model $MODEL_PATH \
    --trust-remote-code \
    --disable-overlap-schedule \
    --cuda-graph-max-bs 54  \
    --tp-size $((WORLD_SIZE * 8)) \
    --ep-size $((WORLD_SIZE * 8)) \
    --dp-size $((WORLD_SIZE * 8)) \
    --enable-dp-lm-head \
    --enable-dp-attention \
    --moe-dense-tp-size 1 \
    --moe-a2a-backend deepep \
    --deepep-mode low_latency \
    --mem-fraction-static 0.86 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --tokenizer-backend fastokens \
    --linear-attn-backend flashinfer \
    --moe-runner-backend auto \
    --max-running-requests 1728 \
    --dist-init-addr ${MASTER_IP}:5302 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size -1 \
    --port ${SGLANG_PORT} \
    --host "${SGLANG_HOST}" \
    --decode-log-interval 1 \
    --prefill-round-robin-balance \
    --disaggregation-mode decode \
    --disaggregation-ib-device "$SGLANG_DISAGGREGATION_IB_DEVICES" \
    --tool-call-parser qwen3_coder \
    > "${LOG_DIR}/coder_decode_server_${NODE_RANK}.log" 2>&1 &

Router 启动脚本 router.sh

PREFILL_IP="http://${1}:$SGLANG_PREFILL_PORT"
DECODE_IP="http://${2}:$SGLANG_DECODE_PORT"
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="${3:-$WORK_HOME/output/$CURRENT_TIME/}"
mkdir -p "$LOG_DIR"

lsof -ti:"$SGLANG_ROUTER_PORT" | xargs -r kill -9

ulimit -n 65535
nohup "$SGLANG_PYTHON" -m sglang_router.launch_router \
    --pd-disaggregation \
    --prefill "$PREFILL_IP" \
    --decode "$DECODE_IP" \
    --host "${SGLANG_HOST}" \
    --mini-lb \
    --request-timeout-secs 7200 \
    --worker-startup-timeout-secs 600 \
    --port "$SGLANG_ROUTER_PORT" > "${LOG_DIR}/coder_router.log" 2>&1 &

Qwen3-235B-A22B-Instruct-2507​

模型链接（Qwen3-235B-A22B-Instruct-2507）

模型链接（EAGLE3-Qwen3-235B-A22B-Instruct-2507-FP8）

1P2D 分离部署​

提示

以下以FP8模型为例，如果是BF16版本需要在prefill_server.sh 以及decode_server.sh添加/修改环境变量 export SGLANG_DEEPEP_BF16_DISPATCH=1

启动方法：将Prefill启动脚本，Decode启动脚本，Router启动脚本，run_sglang.sh和resolve_sglang_pd_ib_devices.sh放在同一路径下。并将需要使用的节点IP放进hostfile文件中。运行bash run_sglang.sh。

一键运行脚本 run.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SSH_PORT="${SSH_PORT:-62216}"

WORKSPACE=$(pwd)
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="$WORK_HOME/output/$CURRENT_TIME/"
PREFILL_SERVER_COUNT=${PREFILL_SERVER_COUNT:-1}
DECODER_SERVER_COUNT=${DECODER_SERVER_COUNT:-2}
PREFILL_NNODES=${PREFILL_NNODES:-$PREFILL_SERVER_COUNT}
DECODER_NNODES=${DECODER_NNODES:-$DECODER_SERVER_COUNT}
REQUIRED_NODES=$((PREFILL_SERVER_COUNT + DECODER_SERVER_COUNT))
MODEL_PATH=/data/models/Qwen3-235B-A22B-Instruct-2507-FP8/
mkdir -p "$LOG_DIR"

# 检查 hostfile 是否存在
if [ ! -f "$WORKSPACE/hostfile" ]; then
    echo "Error: $WORKSPACE/hostfile not found!"
    exit 1
fi

# 读取 hostfile 到数组
mapfile -t hosts < "$WORKSPACE/hostfile"

# 检查是否读取到足够的节点
if [[ ${#hosts[@]} -lt "$REQUIRED_NODES" ]]; then
    echo "Error: hostfile should contain at least $REQUIRED_NODES nodes"
    exit 1
fi

echo "Using hosts:"
printf '%s\n' "${hosts[@]}"
echo "MODEL_PATH: $MODEL_PATH"

PREFILL_MASTER_IP=${hosts[0]}
DECODER_MASTER_IP=${hosts[$PREFILL_SERVER_COUNT]}

# prefill
for ((prefill_rank = 0; prefill_rank < PREFILL_SERVER_COUNT; prefill_rank++)); do
    prefill_host=${hosts[$prefill_rank]}
    ssh -p "$SSH_PORT" "$prefill_host" "cd $WORKSPACE && SGLANG_VENV=$SGLANG_VENV SGLANG_PYTHON=$SGLANG_PYTHON bash prefill_server.sh $PREFILL_MASTER_IP $prefill_rank $PREFILL_NNODES $LOG_DIR $MODEL_PATH" &
done
sleep 3

# decode
for ((decode_rank = 0; decode_rank < DECODER_SERVER_COUNT; decode_rank++)); do
    decode_host_index=$((PREFILL_SERVER_COUNT + decode_rank))
    decode_host=${hosts[$decode_host_index]}
    ssh -p "$SSH_PORT" "$decode_host" "cd $WORKSPACE && SGLANG_VENV=$SGLANG_VENV SGLANG_PYTHON=$SGLANG_PYTHON bash decode_server.sh $DECODER_MASTER_IP $decode_rank $DECODER_NNODES $LOG_DIR $MODEL_PATH" &
    if [[ "$decode_rank" -eq 0 && "$DECODER_SERVER_COUNT" -gt 1 ]]; then
        sleep 5
    else
        sleep 2
    fi
done

# router - 在第一个节点上启动 router
ssh -p "$SSH_PORT" "$PREFILL_MASTER_IP" "cd $WORKSPACE && SGLANG_VENV=$SGLANG_VENV SGLANG_PYTHON=$SGLANG_PYTHON bash router.sh $PREFILL_MASTER_IP $DECODER_MASTER_IP $LOG_DIR" &

echo "All services started in background"
echo "All logs are stored in $LOG_DIR"

prefill启动脚本 prefill_server.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

export TP_SOCKET_IFNAME=bond0
export GLOO_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
# export PYTORCH_MUSA_ALLOC_CONF=expandable_segments:True

export MUSA_LAUNCH_BLOCKING=0

export MCCL_PROTOS=2
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0

export MC_TE_METRIC=1
export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_DEEP_GEMM_BLOCK_M=256
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true
export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_DISAGGREGATION_MAPPING_IB_DEVICE_TO_GPU=1
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SGLANG_IB_TARGET_COUNT=8
SGLANG_IB_REPEAT_PER_DEVICE=2
SGLANG_PD_IB_RESOLVER="${SCRIPT_DIR}/resolve_sglang_pd_ib_devices.sh"
if [[ -f "$SGLANG_PD_IB_RESOLVER" ]]; then
    source "$SGLANG_PD_IB_RESOLVER"
else
    SGLANG_DISAGGREGATION_IB_DEVICES="${SGLANG_DISAGGREGATION_IB_DEVICES:-${SGLANG_IB_DEFAULT_DEVICES:-}}"
    export SGLANG_DISAGGREGATION_IB_DEVICES
    echo "Warning: ${SGLANG_PD_IB_RESOLVER} not found, use default IB devices"
    echo "IB_DEVICES=$SGLANG_DISAGGREGATION_IB_DEVICES"
fi


export VLLM_CONFIGURE_LOGGING=0
export MC_TE_METRIC=true

# export NVSHMEM_IB_TRAFFIC_CLASS=163
# export MUSA_DEVICE_PAGE_SIZE=0x1000
# export MUSA_MANAGED_FORCE_DEVICE_ALLOC=1
# export NVSHMEM_IBGDA_NIC_HANDLER=cpu

sleep 3

SGLANG_PORT="${SGLANG_PORT:-30133}"
MASTER_IP=$1
NODE_RANK=$2
NNODES=$3
WORLD_SIZE=$NNODES
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="${4:-$WORK_HOME/output/$CURRENT_TIME/}"
MODEL_PATH="${5:?MODEL_PATH must be passed from run_sglang.sh}"
mkdir -p "$LOG_DIR"

SHARED_DIR="${SGLANG_SHARED_DIR:-${SCRIPT_DIR}/shared}"
if [[ -z "${SGLANG_SHARED_DIR:-}" && ! -d "$SHARED_DIR" ]]; then
    for shared_candidate in "${SCRIPT_DIR}/../shared" "${SCRIPT_DIR}/../../shared" "${SCRIPT_DIR}/../../../shared" "${SCRIPT_DIR}/../../../../shared"; do
        if [[ -d "$shared_candidate" ]]; then
            SHARED_DIR="$(cd "$shared_candidate" && pwd)"
            break
        fi
    done
fi

DEEP_EP_CONFIG="/data/workspace/deepep/deepep.config"

EPLB_FILE="/data/workspace/eplb/Qwen3-235B-A22B-Instruct-2507/prefill_eplb_ep8.pt"
echo "SHARED_DIR=$SHARED_DIR"
echo "DEEP_EP_CONFIG=$DEEP_EP_CONFIG"
echo "EPLB_FILE=$EPLB_FILE"

export SGLANG_TORCH_PROFILER_DIR="$WORK_HOME/traces_P${NODE_RANK}"
# FP8配置如下
export SGLANG_DEEPEP_BF16_DISPATCH=0
# BF16配置如下：
# export SGLANG_DEEPEP_BF16_DISPATCH=1

nohup "$SGLANG_PYTHON" -m sglang.launch_server \
    --disable-piecewise-cuda-graph \
    --model $MODEL_PATH \
    --trust-remote-code \
    --disable-cuda-graph \
    --disable-overlap-schedule \
    --tp-size 8 \
    --ep-size 8 \
    --dp-size 8 \
    --pp-size 1 \
    --enable-dp-lm-head \
    --enable-dp-attention \
    --moe-dense-tp-size 1 \
    --moe-a2a-backend deepep \
    --moe-runner-backend auto \
    --deepep-mode normal \
    --mem-fraction-static 0.90 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --tokenizer-backend fastokens \
    --linear-attn-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:5303 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size -1 \
    --enable-cache-report \
    --deepep-config "$DEEP_EP_CONFIG" \
    --schedule-conservativeness 1 \
    --max-running-requests 64 \
    --port ${SGLANG_PORT} \
    --host "${SGLANG_HOST}" \
    --init-expert-location "$EPLB_FILE" \
    --load-balance-method round_robin \
    --disaggregation-mode prefill \
    --disaggregation-ib-device "$SGLANG_DISAGGREGATION_IB_DEVICES" \
    --reasoning-parser qwen3 \
    --tool-call-parser qwen25 \
    > "${LOG_DIR}/prefill_server_${NODE_RANK}.log" 2>&1 &

# --speculative-draft-model /data/models/nv-community-Qwen3-235B-A22B-Eagle3/ \
# --speculative-algorithm EAGLE \
# --speculative-num-steps 1 \
# --speculative-eagle-topk 1 \
# --speculative-num-draft-tokens 2 \
# --context-length 131072 \
# --json-model-override-args '{"rope_scaling":{"rope_type":"yarn","factor":4.0,"original_max_position_embeddings":131072}}' \

decoder启动脚本 decoder_server.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

export TP_SOCKET_IFNAME=bond0
export GLOO_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
# export PYTORCH_MUSA_ALLOC_CONF=expandable_segments:True

export MUSA_LAUNCH_BLOCKING=0

export MCCL_PROTOS=2
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0

export MC_TE_METRIC=1
export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEPEP_LL_USE_NVLINK=1
# export SGLANG_DEEPEP_LL_DISABLE_RECV_HOOK=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true
export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_DISAGGREGATION_MAPPING_IB_DEVICE_TO_GPU=1
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SGLANG_IB_TARGET_COUNT=8
SGLANG_IB_REPEAT_PER_DEVICE=2
SGLANG_PD_IB_RESOLVER="${SCRIPT_DIR}/resolve_sglang_pd_ib_devices.sh"
if [[ -f "$SGLANG_PD_IB_RESOLVER" ]]; then
    source "$SGLANG_PD_IB_RESOLVER"
else
    SGLANG_DISAGGREGATION_IB_DEVICES="${SGLANG_DISAGGREGATION_IB_DEVICES:-${SGLANG_IB_DEFAULT_DEVICES:-}}"
    export SGLANG_DISAGGREGATION_IB_DEVICES
    echo "Warning: ${SGLANG_PD_IB_RESOLVER} not found, use default IB devices"
    echo "IB_DEVICES=$SGLANG_DISAGGREGATION_IB_DEVICES"
fi

# export SGLANG_TORCH_PROFILER_DIR=/data/workspace/traces
# export SGLANG_EXPERT_DISTRIBUTION_RECORDER_DIR=/data/workspace/traces
export VLLM_CONFIGURE_LOGGING=0
export MC_TE_METRIC=true
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1

# export NVSHMEM_IB_TRAFFIC_CLASS=136
# export NVSHMEM_IB_TRAFFIC_CLASS=163
# export MUSA_DEVICE_PAGE_SIZE=0x1000
# export MUSA_MANAGED_FORCE_DEVICE_ALLOC=1
# export NVSHMEM_IBGDA_NIC_HANDLER=cpu
# export MUSA_EXECUTION_TIMEOUT=15000

#export SGLANG_ENABLE_SPEC_V2=1
#export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
sleep 3

SGLANG_PORT="${SGLANG_PORT:-30133}"
MASTER_IP=$1
NODE_RANK=$2
NNODES=$3
WORLD_SIZE=$NNODES
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="${4:-$WORK_HOME/output/$CURRENT_TIME/}"
MODEL_PATH="${5:?MODEL_PATH must be passed from run_sglang.sh}"
mkdir -p "$LOG_DIR"

SHARED_DIR="${SGLANG_SHARED_DIR:-${SCRIPT_DIR}/shared}"
if [[ -z "${SGLANG_SHARED_DIR:-}" && ! -d "$SHARED_DIR" ]]; then
    for shared_candidate in "${SCRIPT_DIR}/../shared" "${SCRIPT_DIR}/../../shared" "${SCRIPT_DIR}/../../../shared" "${SCRIPT_DIR}/../../../../shared"; do
        if [[ -d "$shared_candidate" ]]; then
            SHARED_DIR="$(cd "$shared_candidate" && pwd)"
            break
        fi
    done
fi

EPLB_FILE="/data/workspace/eplb/Qwen3-235B-A22B-Instruct-2507/decode_eplb_ep16.pt"
echo "SHARED_DIR=$SHARED_DIR"
echo "EPLB_FILE=$EPLB_FILE"

# FP8配置如下
export SGLANG_DEEPEP_BF16_DISPATCH=0
# BF16配置如下：
# export SGLANG_DEEPEP_BF16_DISPATCH=1

export SGLANG_TORCH_PROFILER_DIR="$WORK_HOME/traces_D${NODE_RANK}"

nohup "$SGLANG_PYTHON" -m sglang.launch_server \
    --model $MODEL_PATH \
    --trust-remote-code \
    --disable-overlap-schedule \
    --cuda-graph-max-bs 64  \
    --disable-piecewise-cuda-graph \
    --tp-size $((WORLD_SIZE * 8)) \
    --ep-size $((WORLD_SIZE * 8)) \
    --dp-size $((WORLD_SIZE * 8)) \
    --enable-dp-lm-head \
    --enable-dp-attention \
    --moe-dense-tp-size 1 \
    --moe-a2a-backend deepep \
    --deepep-mode low_latency \
    --mem-fraction-static 0.88 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --moe-runner-backend auto \
    --max-running-requests 512 \
    --dist-init-addr ${MASTER_IP}:5302 \
    --nnodes ${WORLD_SIZE} \
    --init-expert-location "$EPLB_FILE" \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size -1 \
    --port ${SGLANG_PORT} \
    --host "${SGLANG_HOST}" \
    --speculative-draft-model /data/models/EAGLE3-Qwen3-235B-A22B-Instruct-2507-FP8/ \
    --speculative-algorithm EAGLE3 \
    --speculative-num-steps 1 \
    --speculative-eagle-topk 1 \
    --speculative-num-draft-tokens 2 \
    --decode-log-interval 1 \
    --prefill-round-robin-balance \
    --disaggregation-mode decode \
    --disaggregation-ib-device "$SGLANG_DISAGGREGATION_IB_DEVICES" \
    --reasoning-parser qwen3 \
    --tool-call-parser qwen25 \
    > "${LOG_DIR}/decode_server_${NODE_RANK}.log" 2>&1 &

# --speculative-draft-model /data/models/nv-community-Qwen3-235B-A22B-Eagle3/ \
# --speculative-algorithm EAGLE \
# --speculative-num-steps 1 \
# --speculative-eagle-topk 1 \
# --speculative-num-draft-tokens 2 \
# --context-length 131072 \
# --json-model-override-args '{"rope_scaling":{"rope_type":"yarn","factor":4.0,"original_max_position_embeddings":131072}}' \

Router 启动脚本 router.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"

SGLANG_PREFILL_PORT="${SGLANG_PREFILL_PORT:-30133}"
SGLANG_DECODE_PORT="${SGLANG_DECODE_PORT:-30133}"
SGLANG_ROUTER_PORT="${SGLANG_ROUTER_PORT:-31000}"
PREFILL_IP="http://${1}:$SGLANG_PREFILL_PORT"
DECODE_IP="http://${2}:$SGLANG_DECODE_PORT"
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="${3:-$WORK_HOME/output/$CURRENT_TIME/}"
mkdir -p "$LOG_DIR"

lsof -ti:"$SGLANG_ROUTER_PORT" | xargs -r kill -9

nohup "$SGLANG_PYTHON" -m sglang_router.launch_router \
    --pd-disaggregation \
    --prefill "$PREFILL_IP" \
    --decode "$DECODE_IP" \
    --host "${SGLANG_HOST}" \
    --mini-lb \
    --request-timeout-secs 7200 \
    --worker-startup-timeout-secs 600 \
    --port "$SGLANG_ROUTER_PORT" > "${LOG_DIR}/coder_router.log" 2>&1 &

Qwen3-30B-A3B​

模型链接（Qwen3-30B-A3B） 单机TP4 部署，以FP8模型为例

启动脚本 start_server.sh

#!/usr/bin/env bash
set -euo pipefail

SGLANG_VENV="${SGLANG_VENV:-/root/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
if [[ -f "${SGLANG_VENV}/bin/activate" ]]; then
  # shellcheck disable=SC1091
  source "${SGLANG_VENV}/bin/activate"
fi

export PATH="${SGLANG_VENV}/bin:${PATH}"
export LD_LIBRARY_PATH="/usr/local/musa/lib:/usr/local/openmpi/lib:/usr/local/musa/mudnn/lib:/usr/local/mtshmem/lib:${LD_LIBRARY_PATH:-}"
export MUSA_VISIBLE_DEVICES="${MUSA_VISIBLE_DEVICES:-0}"
export MTHREADS_VISIBLE_DEVICES="${MTHREADS_VISIBLE_DEVICES:-0}"
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_USE_DEEPGEMM_BMM=0
export SGLANG_MOE_PADDING=1
export SGLANG_USE_MUSA_FUSED_KERNEL=1

WORK_HOME="${WORK_HOME:-$PWD}"
MODEL_PATH=/data/models/Qwen3-30B-A3B/
MODEL_NAME="${MODEL_NAME:-Qwen3-8B-FP8}"
PORT="${PORT:-${SGLANG_PORT:-42251}}"
DIST_PORT="${DIST_PORT:-1264}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"
LOG_DIR="${LOG_DIR:-${WORK_HOME}/logs/service}"
SPEC_MODEL_PATH="${SPEC_MODEL_PATH:-/models/Qwen3-8B_eagle3}"
ENABLE_SPECULATIVE="${ENABLE_SPECULATIVE:-1}"
TOKENIZER_BACKEND="${TOKENIZER_BACKEND:-}"

SGLANG_ENABLE_SPECULATIVE="${SGLANG_ENABLE_SPECULATIVE:-${ENABLE_SPECULATIVE}}"
export SGLANG_ENABLE_SPEC_V2="${SGLANG_ENABLE_SPEC_V2:-0}"
mkdir -p "${LOG_DIR}" "${WORK_HOME}/traces"
TP_SIZE="${TP_SIZE:-1}"

MASTER_IP="$(hostname -I | awk '{print $1}')"
LOG_FILE="${LOG_DIR}/qwen3_8b_2500_1500_short_$(date +%Y%m%d_%H%M%S).log"

SPEC_ARGS=()
if [[ "${SGLANG_ENABLE_SPECULATIVE}" == "1" || "${SGLANG_ENABLE_SPECULATIVE}" == "true" ]]; then
  export SGLANG_ENABLE_SPEC_V2=1
  SPEC_ARGS=(
    --speculative-draft-model-path "${SPEC_MODEL_PATH}"
    --speculative-algorithm EAGLE3
    --speculative-num-steps 3
    --speculative-eagle-topk 1
    --speculative-num-draft-tokens 4
  )
fi

TOKENIZER_ARGS=()
if [[ -n "${TOKENIZER_BACKEND}" ]]; then
  TOKENIZER_ARGS=(--tokenizer-backend "${TOKENIZER_BACKEND}")
fi

nohup "${SGLANG_PYTHON}" -m sglang.launch_server \
  --model-path "${MODEL_PATH}" \
  --served-model-name "${MODEL_NAME}" \
  --trust-remote-code \
  --tp-size "${TP_SIZE}" \
  --pp-size 1 \
  --host "${SGLANG_HOST}" \
  --port "${PORT}" \
  --dist-init-addr "${MASTER_IP}:${DIST_PORT}" \
  --attention-backend fa3 \
  --mm-attention-backend fa3 \
  --tokenizer-backend fastokens \
  --linear-attn-backend flashinfer \
  --sampling-backend flashinfer \
  "${TOKENIZER_ARGS[@]}" \
  --mem-fraction-static 0.85 \
  --cuda-graph-bs $(seq 1 64) \
  --disable-piecewise-cuda-graph \
  --chunked-prefill-size -1 \
  --max-running-requests 256 \
  --context-length 12000 \
  --decode-log-interval 1 \
  "${SPEC_ARGS[@]}" \
  --reasoning-parser qwen3 \
  --tool-call-parser qwen25 \
  > "${LOG_FILE}" 2>&1 &

echo "SGLang Qwen3-8B started, pid=$!, log=${LOG_FILE}"

Qwen3-Next-80B-A3B-Instruct​

模型链接（Qwen3-Next-80B-A3B-Instruct） 单机PD分离部署，P：TP4 D：TP4。以FP8模型为例

启动方法：将run_sglang.sh和resolve_sglang_pd_ib_devices.sh放在同一路径下。运行bash run_sglang.sh。

启动脚本 start_server.sh

#!/bin/bash
#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0

export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEP_GEMM_BLOCK_M=256
# 如果模型为 BF16 数据类型则打开该环境变量
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true

export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600

export VLLM_PATCH_MUSA_CUSTOM_OPS=1
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SHARED_DIR="${SGLANG_SHARED_DIR:-${SCRIPT_DIR}/shared}"
if [[ -z "${SGLANG_SHARED_DIR:-}" && ! -d "$SHARED_DIR" ]]; then
    for shared_candidate in "${SCRIPT_DIR}/../shared" "${SCRIPT_DIR}/../../shared" "${SCRIPT_DIR}/../../../shared" "${SCRIPT_DIR}/../../../../shared"; do
        if [[ -d "$shared_candidate" ]]; then
            SHARED_DIR="$(cd "$shared_candidate" && pwd)"
            break
        fi
    done
fi

DEEP_EP_CONFIG="/shared_configs/deepep/deepep.config"

SGLANG_PREFILL_PORT="${SGLANG_PREFILL_PORT:-32011}"
SGLANG_DECODE_PORT="${SGLANG_DECODE_PORT:-32024}"
SGLANG_ROUTER_PORT="${SGLANG_ROUTER_PORT:-35124}"

MASTER_IP=127.0.0.1
NODE_RANK=0
WORLD_SIZE=1
PREFILL_GPU_IDS="${PREFILL_GPU_IDS:-0,1,2,3}"
DECODE_GPU_IDS="${DECODE_GPU_IDS:-4,5,6,7}"
MODEL_PATH="${MODEL_PATH:-/data/models/Qwen3-Next-80B-A3B-Instruct-FP8}"

CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="$WORK_HOME/output/$CURRENT_TIME/"
mkdir -p "$LOG_DIR"
times_now=$(date "+%Y%m%d_%H%M")

PREFILL_LOG_FILE="${LOG_DIR}/${times_now}_prefill.log"
DECODE_LOG_FILE="${LOG_DIR}/${times_now}_decode.log"
ROUTER_LOG_FILE="${LOG_DIR}/${times_now}_reouter.log"

SGLANG_IB_TARGET_COUNT=0
SGLANG_IB_REPEAT_PER_DEVICE=1
SGLANG_PD_IB_RESOLVER="${SCRIPT_DIR}/resolve_sglang_pd_ib_devices.sh"
if [[ -f "$SGLANG_PD_IB_RESOLVER" ]]; then
    source "$SGLANG_PD_IB_RESOLVER"
else
    SGLANG_DISAGGREGATION_IB_DEVICES="${SGLANG_DISAGGREGATION_IB_DEVICES:-${SGLANG_IB_DEFAULT_DEVICES:-}}"
    export SGLANG_DISAGGREGATION_IB_DEVICES
    echo "Warning: ${SGLANG_PD_IB_RESOLVER} not found, use default IB devices"
    echo "IB_DEVICES=$SGLANG_DISAGGREGATION_IB_DEVICES"
fi

export SGLANG_TORCH_PROFILER_DIR="${SCRIPT_DIR}/traces"
export MUSA_VISIBLE_DEVICES="$PREFILL_GPU_IDS"
nohup "$SGLANG_PYTHON" -m sglang.launch_server \
    --disable-piecewise-cuda-graph \
    --model-path "$MODEL_PATH" \
    --trust-remote-code \
    --disable-cuda-graph \
    --disable-overlap-schedule \
    --deepep-config "$DEEP_EP_CONFIG" \
    --tp-size 4 \
    --ep-size 4 \
    --dp-size 4 \
    --enable-dp-lm-head \
    --enable-dp-attention \
    --moe-dense-tp-size 1 \
    --moe-a2a-backend deepep \
    --deepep-mode normal \
    --mem-fraction-static 0.85 \
    --attention-backend fa3 \
    --linear-attn-backend flashinfer \
    --tokenizer-backend fastokens \
    --moe-runner-backend auto \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:2453 \
    --nnodes "$WORLD_SIZE" \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size -1 \
    --mamba-scheduler-strategy extra_buffer \ 
    --enable-cache-report \
    --schedule-conservativeness 1 \
    --max-running-requests 64 \
    --max-prefill-tokens 4096 \
    --ep-num-redundant-experts 0 \
    --port ${SGLANG_PREFILL_PORT} \
    --host "${SGLANG_HOST}" \
    --load-balance-method round_robin \
    --disaggregation-mode prefill \
    --reasoning-parser qwen3 \
    --tool-call-parser qwen \	
    --disaggregation-ib-device "$SGLANG_DISAGGREGATION_IB_DEVICES" > "$PREFILL_LOG_FILE" 2>&1 &
sleep 10

export SGLANG_DEEPEP_LL_USE_NVLINK=1
# export SGLANG_DEEPEP_LL_DISABLE_RECV_HOOK=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128

export MUSA_VISIBLE_DEVICES="$DECODE_GPU_IDS"
nohup "$SGLANG_PYTHON" -m sglang.launch_server \
    --disable-piecewise-cuda-graph \
    --model-path "$MODEL_PATH" \
    --trust-remote-code \
    --cuda-graph-max-bs 128  \
    --tp-size 4 \
    --ep-size 4 \
    --dp-size 4 \
    --enable-dp-lm-head \
    --enable-dp-attention \
    --moe-dense-tp-size 1 \
    --moe-a2a-backend deepep \
    --deepep-mode low_latency \
    --mem-fraction-static 0.8 \
    --attention-backend fa3 \
    --linear-attn-backend flashinfer \
    --tokenizer-backend fastokens \
    --moe-runner-backend auto \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --max-running-requests 128 \
    --dist-init-addr ${MASTER_IP}:1122 \
    --nnodes "$WORLD_SIZE" \
    --node-rank ${NODE_RANK} \
    --ep-num-redundant-experts 0 \
    --chunked-prefill-size -1 \
    --port ${SGLANG_DECODE_PORT} \
    --host "${SGLANG_HOST}" \
    --decode-log-interval 1 \
    --load-balance-method round_robin \
    --prefill-round-robin-balance \
    --disaggregation-mode decode \
    --reasoning-parser qwen3 \
    --tool-call-parser qwen \	
    --disaggregation-ib-device "$SGLANG_DISAGGREGATION_IB_DEVICES" > "$DECODE_LOG_FILE" 2>&1 &



lsof -ti:"$SGLANG_ROUTER_PORT" | xargs -r kill -9


PREFILL_IP="http://127.0.0.1:$SGLANG_PREFILL_PORT"
DECODE_IP="http://127.0.0.1:$SGLANG_DECODE_PORT"

nohup "$SGLANG_PYTHON" -m sglang_router.launch_router \
    --pd-disaggregation \
    --mini-lb \
    --prefill $PREFILL_IP \
    --decode $DECODE_IP \
    --host "${SGLANG_HOST}" \
    --request-timeout-secs 7200 \
    --port $SGLANG_ROUTER_PORT > "$ROUTER_LOG_FILE" 2>&1 &

while [ 0 -eq 0 ];do
	grep "The server is fired up and ready to roll!" "$DECODE_LOG_FILE" > /dev/null 2>&1
	if [ $? -eq 0 ];then
		break
	fi
done
echo "All logs are stored in $LOG_DIR"


版权所有 2024 摩尔线程，本文档受国际版权法保护。

摩尔线程和摩尔线程徽标是摩尔线程智能科技 (北京)有限责任公司的注册商标。

免责声明

本文档提供有关摩尔线程产品的信息。本文档并未授权任何知识产权的许可，并未以明示或暗示，或以禁止反言或其他方式授予任何知识产权许可。