跳到主要内容

GLM 模型部署

GLM-5.2 模型部署​

本章节为 GLM-5.2-FP8 部署指导，完整示例方案是2P4D，即2台机器形成1个 Prefill 实例，4台机器形成1个 Decode 实例。结尾也提供了 3P4D 部署方案的 Prefill 节点启动脚本示例，更适合长上下文场景 。 用户可根据业务要求选择合适的部署方式。

  • 脚本涉及的参数说明和一键运行说明见 快速开始。
注意

注意 GLM-5.2 需要使用以下单独的镜像，不能使用 sglang:v0.5.6.post3 统一镜像。

# Intel/AMD 平台
docker pull registry.mthreads.com/devtech/glm5.2:1.2.0

Prefill启动（1个 Prefill 实例含2个节点，并行策略 PP2CP8EP8）​

prefill启动脚本 prefill_server.sh

# P节点
#!/bin/bash
# set -x
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

pip uninstall -y sglang >/dev/null 2>&1 || true
export PYTHONPATH=/workspace/sglang/python:$PYTHONPATH

# Non-MTT stack: use sgl-kernel MUSA FP8 GEMM + tilelang DSA + DeepGEMM indexer
export SGLANG_USE_MTT_INDEXER=0
export SGLANG_USE_MTT_ATTN=0
export SGLANG_USE_MTT_F8GEMM=0
export SGLANG_USE_MTT_HGEMM=0
export SGLANG_USE_MTT_EPI_QKV=0
export SGLANG_NSA_FUSE_TOPK=1
export SGLANG_OPT_USE_TRITON_MASKED_SWIGLU_QUANT=1
export SGLANG_OPT_FUSED_INDEXER_QK_NORM_ROPE=1
export SGLANG_FUSED_MLA_ROPE_CACHE=1

export MCCL_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export GLOO_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
# export PYTORCH_MUSA_ALLOC_CONF=expandable_segments:True

# export MUSA_BLOCK_SCHEDULE_MODE=1
# export MUSA_USERQ=1
export SGLANG_DEEP_GEMM_BLOCK_M=256
export MUSA_PRINT_ENV=0
export MUSA_LAUNCH_BLOCKING=0
export MUSA_EXECUTION_TIMEOUT=900000
export DEEPEP_CPU_POLL_TIMEOUT=200
export MUSA_ERROR_DUMP_VERBOSE=1
export MUSA_ENABLE_LLC_OPT=1

export MCCL_PROTOS=2
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_IB_DISABLE=0
export MCCL_P2P_LEVEL=MT2
# export NVSHMEM_IB_TRAFFIC_CLASS=136

# mooncake
export MC_TE_METRIC=1
export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEPEP_USE_MUSA_ACE=1
export SGLANG_SBO_COMBINE_SHARED=0
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true
export SGLANG_ENABLE_SPEC_V2=true

# Mooncake KV transfer (prefill-side senders).
export SGLANG_DISAGGREGATION_QUEUE_SIZE=4
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=8
export SGLANG_DISAGGREGATION_ALL_CP_RANKS_TRANSFER=1
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=600

# AUX metadata over TCP; avoids last-PP-stage AUX RDMA head-of-line blocking.
export SGLANG_MOONCAKE_SEND_AUX_TCP=1
export SGLANG_DISAGGREGATION_MAPPING_IB_DEVICE_TO_GPU=1
export SGLANG_HEALTH_CHECK_TIMEOUT=600

export SGLANG_FUSED_MLA_ROPE_CACHE=1
# export MUSA_EXECUTION_TIMEOUT=900000
export SGLANG_TORCH_PROFILER_DIR=$SCRIPT_DIR/traces
export SGLANG_EXPERT_DISTRIBUTION_RECORDER_DIR=$SCRIPT_DIR/traces
DEEP_EP_CONFIG="/workspace/code/deepep.config"


fp8_gemm_backend="auto"
if [ $SGLANG_USE_MTT_F8GEMM -gt 0 ]; then
  fp8_gemm_backend="mtt"
fi

nsa_backend="tilelang"
if [ $SGLANG_USE_MTT_ATTN -gt 0 ]; then
  nsa_backend="mtt_dsa"
fi

# IB 设备自动检测（200G / 400G）
IB_DEVS=()
for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        rate=$(cat "$port/rate" 2>/dev/null)
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
SGLANG_IB_DEVICES=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
export MCCL_IB_HCA="$SGLANG_IB_DEVICES"


NODE_RANK=$1
PREFILL_IP="${2}:4343"
LOG_DIR=$3
MODEL_PATH=$4
HOST_IP=$5
WORLD_SIZE=$6
SGLANG_PORT=24586

mkdir -p "${LOG_DIR}"


PYTHON_ARGS=()

ENABLE_CP=1
if [ $ENABLE_CP -gt 0 ]; then
    PYTHON_ARGS+=(--dp-size 1)
    PYTHON_ARGS+=(--enable-nsa-prefill-context-parallel)
    sglang_dump_dir="saved_pp4cp8"
else
    PYTHON_ARGS+=(--dp-size 8)
    PYTHON_ARGS+=(--enable-dp-attention)
    sglang_dump_dir="saved_pp4dp8"
fi

export SGLANG_OPT_FLASHMLA_SPARSE_PREFILL=0
export SGLANG_DUMPER_ENABLE=0
export SGLANG_DUMPER_DIR=$sglang_dump_dir

export SGLANG_PP_LAYER_PARTITION="40,38"

echo "${PYTHON_ARGS[@]}"


nohup python3 -u -m sglang.launch_server \
    --model $MODEL_PATH \
    --trust-remote-code \
    --disable-overlap-schedule \
    --enable-single-batch-overlap \
    --disable-cuda-graph \
    "${PYTHON_ARGS[@]}" \
    --tp-size 8 \
    --ep-size 8 \
    --pp-size $WORLD_SIZE \
    --moe-dense-tp-size 1 \
    --enable-dp-lm-head \
    --fp8-gemm-backend $fp8_gemm_backend \
    --attention-backend dsa \
    --dsa-prefill-backend tilelang \
    --dsa-decode-backend tilelang \
    --kv-cache-dtype fp8_e4m3 \
    --moe-a2a-backend deepep \
    --deepep-mode normal \
    --deepep-config "$DEEP_EP_CONFIG" \
    --sampling-backend flashinfer \
    --sampling-backend flashinfer \
    --mem-fraction-static 0.80 \
    --max-running-requests $((36)) \
    --chunked-prefill-size $((8 * 1024)) \
    --max-prefill-tokens $((256 * 1024)) \
    --context-length $((256 * 1024)) \
    --dist-init-addr ${PREFILL_IP} \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --host 0.0.0.0 \
    --port ${SGLANG_PORT} \
    --log-requests-level 3 \
    --log-level info \
    --tokenizer-backend fastokens \
    --enable-metrics \
    --enable-metrics-for-all-schedulers \
    --export-metrics-to-file \
    --export-metrics-to-file-dir ${LOG_DIR} \
    --enable-cache-report \
    --load-balance-method follow_bootstrap_room \
    --disaggregation-mode prefill \
    --watchdog-timeout 900 \
    --speculative-algorithm EAGLE \
    --speculative-num-steps 5 \
    --speculative-eagle-topk 1 \
    --speculative-num-draft-tokens 6 \
    --disaggregation-ib-device "${SGLANG_IB_DEVICES}" > "${LOG_DIR}/P${NODE_RANK}_${HOST_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &


Decode启动​

decode启动脚本 decode_server.sh

# D节点
#!/bin/bash
# set -x
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

export PYTHONPATH=/workspace/sglang/python:$PYTHONPATH
export SGLANG_NSA_FUSE_TOPK=1
export SGLANG_SYNC_AFTER_RUN_BATCH=0

# --- Operator fusions (validated / known-good on this DeepEP LL decode) ---
export SGLANG_OPT_USE_TRITON_MASKED_SWIGLU_QUANT=1
export SGLANG_OPT_USE_JIT_EP_ACTIVATION=1
export SGLANG_OPT_SWIGLU_CLAMP_FUSION=1
# Keep fused GEMV at tiny-M only (M<=3 == legacy <4); larger M is slower here
export SGLANG_MUSA_FUSED_GEMV_MAX_M=3
# TileKernels paths are prefill-oriented; leave off for decode
export SGLANG_OPT_USE_TILEKERNELS_FP8_QUANT=0
export SGLANG_OPT_USE_TILEKERNELS_SWIGLU_QUANT=0
export SGLANG_MASKED_GEMM_FAST_ACT=0
# Indexer / MLA rope fusions
export SGLANG_OPT_FUSED_INDEXER_QK_NORM_ROPE=1
export SGLANG_FUSED_MLA_ROPE_CACHE=1
export SGLANG_USE_FUSED_METADATA_COPY=1
# MUSA: Triton fused metadata copy (CUDA JIT unavailable).
# Cap MUST cover the longest prompt you serve. zhipu bench uses encoding=65536;
# 16384 truncated page_table → wrong KV → accept rate collapse (~0.60→~0.30).
# PREP_IN_CUDA_GRAPH gathers this width every decode step — 262144 was ~4x
# overkill vs observed max ~131k (p50 still ~65k). 139264 = 128k+11k margin.
export SGLANG_FUSED_METADATA_COPY_BACKEND=triton
export SGLANG_DSA_ENABLE_MTP_PRECOMPUTE_METADATA=1
export SGLANG_DSA_MAX_SEQ_LEN_FOR_CAPTURE=262144
export SGLANG_OPT_USE_JIT_NORM=1

export MCCL_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export GLOO_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MUSA_PRINT_ENV=0
export MUSA_LAUNCH_BLOCKING=0
# export MUSA_EXECUTION_TIMEOUT=12000
export MUSA_ERROR_DUMP_VERBOSE=1
export MUSA_ENABLE_LLC_OPT=1
export SGLANG_FUSE_VERIFY_SAMPLE=1
# Optional MUSA scheduler knobs (lwj experimented with these)
# BLOCK_SCHEDULE_MODE=0 breaks cuda-graph capture (MUSA op not permitted).
export MUSA_BLOCK_SCHEDULE_MODE=1
# USERQ=0: avoid leaving sample/VTG in user queue during host waits.
export MUSA_USERQ=0

export MCCL_PROTOS=2
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0

# mooncake
export MC_TE_METRIC=0
export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_SBO_SHARED_DISPATCH_TWO_STREAM=1
# Retry combine/shared overlap alone (FAST_ACT stays off)
export SGLANG_SBO_COMBINE_SHARED=1
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true
export SGLANG_ENABLE_SPEC_V2=true
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export NVSHMEM_IBGDA_FORCE_NIC_BUF_MEMTYPE=gpumem
export SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_IDLE=0
export SGLANG_DISAGGREGATION_HEARTBEAT_INTERVAL=100
export SGLANG_DISAGGREGATION_QUEUE_SIZE=4
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=8
export SGLANG_DISAGGREGATION_ALL_CP_RANKS_TRANSFER=1
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=1200
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=1200
export SGLANG_DISAGGREGATION_MAPPING_IB_DEVICE_TO_GPU=1
export SGLANG_HEALTH_CHECK_TIMEOUT=600
export SGLANG_PREP_IN_CUDA_GRAPH=1
export SGLANG_DSA_PREP_IN_CUDA_GRAPH=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_DSA_EAGLE_DRAFT_EXTEND_CUDA_GRAPH=1
export SGLANG_LM_HEAD_TP_SIZE=8
# # KV transfer
export SGLANG_MOONCAKE_SEND_AUX_TCP=1
export SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS=128

#export SGLANG_USE_FUSED_METADATA_COPY=0
fp8_gemm_backend="auto"
nsa_backend="tilelang"
IB_DEVS=()

for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")

    for port in "$dev"/ports/*; do
        # 速率（如 200 Gb/sec）
        rate=$(cat "$port/rate" 2>/dev/null)
        # link_layer: InfiniBand / Ethernet (RoCE)
        link=$(cat "$port/link_layer" 2>/dev/null)

        # 只要是 200G（IB 或 RoCE 都收）
        if [[ "$rate" == *"400 Gb"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done

# 排序 + 去重
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)

# 生成 NCCL / SGLang 变量
NCCL_IB_HCA=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
SGLANG_IB_DEVICES="$NCCL_IB_HCA"
echo "IB_DEVICES=$NCCL_IB_HCA"

ENABLE_PROFILER=0
POSITIONAL=()
while [[ $# -gt 0 ]]; do
    case $1 in
        --profiler)
            ENABLE_PROFILER=1
            shift
            ;;
        *)
            POSITIONAL+=("$1")
            shift
            ;;
    esac
done
set -- "${POSITIONAL[@]}"

NODE_RANK=$1
DECODER_IP="${2}:4343"
LOG_DIR=$3
MODEL_PATH=$4
HOST_IP=$5
WORLD_SIZE=$6
SGLANG_PORT=23457

graph_bs=4

export SGLANG_DUMPER_ENABLE=0

PYTHON_ARGS=()
WITH_MTP=1
if [ $WITH_MTP -gt 0 ]; then
    # Known-good: steps=3, draft=4 (TPOT~35). 2/3 measured worse (~58).
    PYTHON_ARGS+=(--speculative-algorithm EAGLE)
    PYTHON_ARGS+=(--speculative-num-steps 5)
    PYTHON_ARGS+=(--speculative-eagle-topk 1)
    PYTHON_ARGS+=(--speculative-num-draft-tokens 6)
fi

echo "${PYTHON_ARGS[@]}"

if [ "$ENABLE_PROFILER" -gt 0 ]; then
    PROFILER_DIR="${LOG_DIR}/profiler/D${NODE_RANK}_${HOST_IP}"
    mkdir -p "$PROFILER_DIR"
    export SGLANG_TORCH_PROFILER_DIR="$PROFILER_DIR"
    export SGLANG_PROFILE_WITH_STACK=${SGLANG_PROFILE_WITH_STACK:-0}
    export SGLANG_PROFILE_RECORD_SHAPES=${SGLANG_PROFILE_RECORD_SHAPES:-1}
    echo "SGLANG_TORCH_PROFILER_DIR=$PROFILER_DIR"
    echo "Profiler: 就绪后手动 start，例如 curl -X POST http://127.0.0.1:${SGLANG_PORT}/start_profile -H 'Content-Type: application/json' -d '{\"start_step\":3,\"num_steps\":30}'"
fi


nohup python3 -u -m sglang.launch_server \
    --model $MODEL_PATH \
    --trust-remote-code \
    --disable-radix-cache \
    --enable-single-batch-overlap \
    --cuda-graph-bs $(seq 1 $graph_bs) \
    "${PYTHON_ARGS[@]}" \
    --tp-size $((WORLD_SIZE * 8)) \
    --ep-size $((WORLD_SIZE * 8)) \
    --dp-size $((WORLD_SIZE * 8)) \
    --enable-dp-attention \
    --moe-dense-tp-size 1 \
    --enable-dp-lm-head \
    --fp8-gemm-backend $fp8_gemm_backend \
    --attention-backend dsa \
    --dsa-prefill-backend tilelang \
    --dsa-decode-backend tilelang \
    --kv-cache-dtype fp8_e4m3 \
    --moe-a2a-backend deepep \
    --deepep-mode low_latency \
    --sampling-backend flashinfer \
    --mem-fraction-static 0.8 \
    --max-running-requests $((WORLD_SIZE * 8 * graph_bs)) \
    --context-length $((256 * 1024)) \
    --reasoning-parser glm45 \
    --tool-call-parser glm47  \
    --dist-init-addr ${DECODER_IP} \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --host 0.0.0.0 \
    --port ${SGLANG_PORT} \
    --log-requests-level 3 \
    --log-level info \
    --enable-metrics \
    --enable-metrics-for-all-schedulers \
    --export-metrics-to-file \
    --export-metrics-to-file-dir ${LOG_DIR} \
    --load-balance-method total_requests \
    --disaggregation-mode decode  \
    --speculative-accept-threshold-single 1.0 \
    --speculative-accept-threshold-acc 1.0 \
    --disaggregation-ib-device $SGLANG_IB_DEVICES > "${LOG_DIR}/D${NODE_RANK}_${HOST_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

此处Decode侧的所有节点应该保持--dist-init-addr参数均为同一个Decode节点的 IP 与端口。

Router启动​

Router启动脚本 router.sh


PREFILL_IP="http://${1}:24586"
DECODE_IP="http://${2}:23457"
LOG_DIR=$3

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export PYTHONPATH=/workspace/sglang/python:$PYTHONPATH
export TORCH_DEVICE_BACKEND_AUTOLOAD=0

export RUST_BACKTRACE=1
ulimit -n 65535
nohup python -m sglang_router.launch_router \
    --pd-disaggregation \
    --prefill $PREFILL_IP \
    --decode $DECODE_IP \
    --host 0.0.0.0 \
    --request-timeout-secs 7200 \
    --prefill-policy round_robin \
    --decode-policy round_robin \
    --port 31000 > "${LOG_DIR}/router_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

创建 hostfile 文件​

该示例为 2P4D 共6台机器，可根据实际部署要求增删IP列表。 hostfile文件示例

192.168.100.101
192.168.100.102
192.168.100.103
192.168.100.104
192.168.100.105
192.168.100.106

一键启动所有服务​

该示例为 2P4D 部署方式，可根据实际部署要求修改 PREFILL_SERVER_COUNT 和 PREFILL_SERVER_COUNT 的参数值。

一键运行脚本 run.sh

#!/bin/bash
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
#echo $CURRENT_TIME
#mkdir -p ./output/$CURRENT_TIME

PREFILL_SERVER_COUNT=${PREFILL_SERVER_COUNT:-2}
DECODER_SERVER_COUNT=${DECODER_SERVER_COUNT:-4}
WORLD_SIZE=$(( (PREFILL_SERVER_COUNT + DECODER_SERVER_COUNT) * 8 ))
port=62216

echo -e "\033[32mPREFILL_SERVER_COUNT: $PREFILL_SERVER_COUNT, DECODER_SERVER_COUNT: $DECODER_SERVER_COUNT, \033[0m"

set -u
  WORK_HOME="$PWD"
  EXPNAME="P${PREFILL_SERVER_COUNT}_D${DECODER_SERVER_COUNT}_gpus${WORLD_SIZE}"
  MODEL_PATH=${MODEL_PATH:-"/data/models/GLM-5.2-FP8"}
  HOSTFILE=./hostfile
  LOG_DIR=$WORK_HOME/output/$EXPNAME/$CURRENT_TIME/
  PREFILL_SCRIPT_FILE="$WORK_HOME/prefill_server.sh"
  DECODER_SCRIPT_FILE="$WORK_HOME/decode_server.sh"
  ROUTER_SCRIPT_FILE="$WORK_HOME/router.sh"
set +u

echo -e "\033[32mWORK_HOME: $WORK_HOME\033[0m"
echo -e "\033[32mMODEL_PATH: $MODEL_PATH\033[0m"
echo -e "\033[32mPREFILL_CRIPT_FILE: $PREFILL_SCRIPT_FILE\033[0m"
echo -e "\033[32mDECODER_SCRIPT_FILE: $DECODER_SCRIPT_FILE\033[0m"

# bash generate_hostfile.sh 
COUNT=0
hostlist=$(grep -v '^#\|^$' $HOSTFILE | awk '{print $1}' | xargs)

# Check if hostlist is empty
if [ -z "$hostlist" ]; then
  echo "Error: hostlist is empty. Please add IP addresses to the hostfile."
  exit 1
fi
PREFILL_MASTER_IP=$(head -n 1 $HOSTFILE)
DECODER_MASTER_IP=$(tail -n $DECODER_SERVER_COUNT $HOSTFILE | head -n1)
echo -e "\033[32mPREFILL_MASTER_IP: $PREFILL_MASTER_IP\033[0m"
echo -e "\033[32mDECODER_MASTER_IP: $DECODER_MASTER_IP\033[0m"


mkdir -p $LOG_DIR
for host in ${hostlist[@]}; do
  echo $host
  if [[ $COUNT -lt $PREFILL_SERVER_COUNT ]]; then
    PREFILL_INDEX=$COUNT
    cmd="bash -c 'bash $PREFILL_SCRIPT_FILE $PREFILL_INDEX $PREFILL_MASTER_IP $LOG_DIR $MODEL_PATH $host $PREFILL_SERVER_COUNT'"
    #cmd="bash -c 'cd $WORK_HOME; echo $PATH '"
    echo $cmd
    ssh -p $port -f -n $host $cmd
  else
    DECODER_INDEX=$((COUNT - PREFILL_SERVER_COUNT))
    cmd="bash -c 'bash $DECODER_SCRIPT_FILE $DECODER_INDEX $DECODER_MASTER_IP $LOG_DIR $MODEL_PATH $host $DECODER_SERVER_COUNT'"
    echo $cmd
    ssh -p $port -f -n $host $cmd
  fi

  ((COUNT++))
done
cmd="bash -c 'bash $ROUTER_SCRIPT_FILE $PREFILL_MASTER_IP $DECODER_MASTER_IP $LOG_DIR'"
echo $cmd
ssh -p $port -f -n $PREFILL_MASTER_IP $cmd
echo -e "Main log file: \033[34m$LOG_DIR\033[0m"

Prefill启动（1个 Prefill 实例含3个节点，并行策略 PP3CP8EP8）​

prefill启动脚本 start_prefill.sh

# P节点
#!/bin/bash
# set -x
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

pip uninstall -y sglang >/dev/null 2>&1 || true
export PYTHONPATH=/workspace/sglang/python:$PYTHONPATH

# Non-MTT stack: use sgl-kernel MUSA FP8 GEMM + tilelang DSA + DeepGEMM indexer
export SGLANG_USE_MTT_INDEXER=0
export SGLANG_USE_MTT_ATTN=0
export SGLANG_USE_MTT_F8GEMM=0
export SGLANG_USE_MTT_HGEMM=0
export SGLANG_USE_MTT_EPI_QKV=0
export SGLANG_NSA_FUSE_TOPK=1
export SGLANG_OPT_USE_TRITON_MASKED_SWIGLU_QUANT=1
export SGLANG_OPT_FUSED_INDEXER_QK_NORM_ROPE=1
export SGLANG_FUSED_MLA_ROPE_CACHE=1

export MCCL_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export GLOO_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
# export PYTORCH_MUSA_ALLOC_CONF=expandable_segments:True

# export MUSA_BLOCK_SCHEDULE_MODE=1
# export MUSA_USERQ=1
export SGLANG_DEEP_GEMM_BLOCK_M=256
export MUSA_PRINT_ENV=0
export MUSA_LAUNCH_BLOCKING=0
export MUSA_EXECUTION_TIMEOUT=900000
export DEEPEP_CPU_POLL_TIMEOUT=200
export MUSA_ERROR_DUMP_VERBOSE=1
export MUSA_ENABLE_LLC_OPT=1

export MCCL_PROTOS=2
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_IB_DISABLE=0
export MCCL_P2P_LEVEL=MT2
# export NVSHMEM_IB_TRAFFIC_CLASS=136

# mooncake
export MC_TE_METRIC=1
export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEPEP_USE_MUSA_ACE=1
export SGLANG_SBO_COMBINE_SHARED=0
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true
export SGLANG_ENABLE_SPEC_V2=true

# Mooncake KV transfer (prefill-side senders).
export SGLANG_DISAGGREGATION_QUEUE_SIZE=4
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=8
export SGLANG_DISAGGREGATION_ALL_CP_RANKS_TRANSFER=1
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=600

# AUX metadata over TCP; avoids last-PP-stage AUX RDMA head-of-line blocking.
export SGLANG_MOONCAKE_SEND_AUX_TCP=1
export SGLANG_DISAGGREGATION_MAPPING_IB_DEVICE_TO_GPU=1
export SGLANG_HEALTH_CHECK_TIMEOUT=600

export SGLANG_FUSED_MLA_ROPE_CACHE=1
# export MUSA_EXECUTION_TIMEOUT=900000
export SGLANG_TORCH_PROFILER_DIR=$SCRIPT_DIR/traces
export SGLANG_EXPERT_DISTRIBUTION_RECORDER_DIR=$SCRIPT_DIR/traces
DEEP_EP_CONFIG="/workspace/code/deepep.config"


fp8_gemm_backend="auto"
if [ $SGLANG_USE_MTT_F8GEMM -gt 0 ]; then
  fp8_gemm_backend="mtt"
fi

nsa_backend="tilelang"
if [ $SGLANG_USE_MTT_ATTN -gt 0 ]; then
  nsa_backend="mtt_dsa"
fi

# IB 设备自动检测（200G / 400G）
IB_DEVS=()
for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        rate=$(cat "$port/rate" 2>/dev/null)
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
SGLANG_IB_DEVICES=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
export MCCL_IB_HCA="$SGLANG_IB_DEVICES"


NODE_RANK=$1
PREFILL_IP="${2}:4343"
LOG_DIR=$3
MODEL_PATH=$4
HOST_IP=$5
WORLD_SIZE=$6
SGLANG_PORT=24586

mkdir -p "${LOG_DIR}"


PYTHON_ARGS=()

ENABLE_CP=1
if [ $ENABLE_CP -gt 0 ]; then
    PYTHON_ARGS+=(--dp-size 1)
    PYTHON_ARGS+=(--enable-nsa-prefill-context-parallel)
    sglang_dump_dir="saved_pp4cp8"
else
    PYTHON_ARGS+=(--dp-size 8)
    PYTHON_ARGS+=(--enable-dp-attention)
    sglang_dump_dir="saved_pp4dp8"
fi

export SGLANG_OPT_FLASHMLA_SPARSE_PREFILL=0
export SGLANG_DUMPER_ENABLE=0
export SGLANG_DUMPER_DIR=$sglang_dump_dir

# export SGLANG_PP_LAYER_PARTITION="40,38"
export SGLANG_PP_LAYER_PARTITION="28,26,24"

echo "${PYTHON_ARGS[@]}"


nohup python3 -u -m sglang.launch_server \
    --model $MODEL_PATH \
    --trust-remote-code \
    --disable-overlap-schedule \
    --enable-single-batch-overlap \
    --disable-cuda-graph \
    "${PYTHON_ARGS[@]}" \
    --tp-size 8 \
    --ep-size 8 \
    --pp-size $WORLD_SIZE \
    --moe-dense-tp-size 1 \
    --enable-dp-lm-head \
    --fp8-gemm-backend $fp8_gemm_backend \
    --attention-backend dsa \
    --dsa-prefill-backend tilelang \
    --dsa-decode-backend tilelang \
    --kv-cache-dtype fp8_e4m3 \
    --moe-a2a-backend deepep \
    --deepep-mode normal \
    --deepep-config "$DEEP_EP_CONFIG" \
    --sampling-backend flashinfer \
    --sampling-backend flashinfer \
    --mem-fraction-static 0.65 \
    --max-running-requests $((16)) \
    --chunked-prefill-size $((14 * 1024)) \
    --max-prefill-tokens $((256 * 1024)) \
    --context-length $((256 * 1024)) \
    --prefill-max-requests 4 \
    --pp-max-micro-batch-size 5 \
    --dist-init-addr ${PREFILL_IP} \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --host 0.0.0.0 \
    --port ${SGLANG_PORT} \
    --log-requests-level 3 \
    --log-level info \
    --tokenizer-backend fastokens \
    --enable-metrics \
    --enable-metrics-for-all-schedulers \
    --export-metrics-to-file \
    --export-metrics-to-file-dir ${LOG_DIR} \
    --enable-cache-report \
    --load-balance-method follow_bootstrap_room \
    --disaggregation-mode prefill \
    --watchdog-timeout 900 \
    --speculative-algorithm EAGLE \
    --speculative-num-steps 5 \
    --speculative-eagle-topk 1 \
    --speculative-num-draft-tokens 6 \
    --disaggregation-ib-device "${SGLANG_IB_DEVICES}" > "${LOG_DIR}/P${NODE_RANK}_${HOST_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

GLM-5/5.1 模型部署​

本章节以 GLM-5-FP8 部署为例，采取的方案是4P4D，即4台机器作为Prefill，4台机器为Decode。最小部署模式为2P4D，注意 256（expert数）需要能被 Decode 机器数整除。该部署方式同样适用于 GLM-5.1。

  • 脚本涉及的参数说明和一键运行说明见 快速开始。
  • 如果需要修改部署方式，请更新 run.sh 中 PREFILL_SERVER_COUNT 和 DECODE_SERVER_COUNT。
  • DeepEP 的配置文件内容参考 DeepEP 配置文件
信息

启动 SGLang 服务前需要切换python虚拟环境，执行命令：workon glm5

Prefill启动​

prefill启动脚本 prefill_server.sh

#!/bin/bash

source /root/.local/bin/uv-virtualenvwrapper.sh
workon glm5

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export PYTHONPATH=/sgl-workspace/sglang-glm5/python:$PYTHONPATH

export MATE_FORCE_JIT=1
export SGLANG_USE_MTT_F8GEMM=1
export SGLANG_USE_MTT_HGEMM=1
export SGLANG_USE_MTT_EPI_QKV=1
export SGLANG_USE_MTT_INDEXER=1
export SGLANG_NSA_FUSE_TOPK=1
export SGLANG_USE_MTT_ATTN=1

export TP_SOCKET_IFNAME=bond0
export GLOO_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
# export PYTORCH_MUSA_ALLOC_CONF=expandable_segments:True

# export MUSA_BLOCK_SCHEDULE_MODE=1
# export MUSA_USERQ=1
export MUSA_PRINT_ENV=0
export MUSA_LAUNCH_BLOCKING=0
# export MUSA_EXECUTION_TIMEOUT=12000
export MUSA_ERROR_DUMP_VERBOSE=1
export MUSA_ENABLE_LLC_OPT=1

export MCCL_PROTOS=2
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0

# mooncake
export MC_TE_METRIC=1
export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEPEP_USE_MUSA_ACE=1
export SGLANG_SBO_COMBINE_SHARED=1 
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true

export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=300
export SGLANG_DISAGGREGATION_MAPPING_IB_DEVICE_TO_GPU=1
export SGLANG_HEALTH_CHECK_TIMEOUT=100

export SGLANG_TORCH_PROFILER_DIR=$SCRIPT_DIR/traces
export SGLANG_EXPERT_DISTRIBUTION_RECORDER_DIR=$SCRIPT_DIR/traces

# 获取当前环境的 RDMA 设备名列表，在启动时作为 disaggregation-ib-device 参数值
IB_DEVS=()

for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        rate=$(cat "$port/rate" 2>/dev/null)
        # link_layer: InfiniBand / Ethernet (RoCE)
        link=$(cat "$port/link_layer" 2>/dev/null)
        state=$(cat "$port/state" 2>/dev/null)
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]] && [[ "$state" == "4"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done

IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)

MCCL_IB_HCA=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
SGLANG_IB_DEVICES="$MCCL_IB_HCA"
echo "IB_DEVICES=$MCCL_IB_HCA"


nsa_backend="tilelang"
if [ $SGLANG_USE_MTT_ATTN -gt 0 ]; then
  nsa_backend="mtt_dsa"
fi

NODE_RANK=$1
PREFILL_IP="${2}:5303"
LOG_DIR=$3
MODEL_PATH=$4
HOST_IP=$5
NNODES=$6

SGLANG_PORT=20133
DEEP_EP_CONFIG="${SCRIPT_DIR}/deepep.config"

mkdir -p "${LOG_DIR}"

rm -rf ~/.triton/cache/

nohup python3 -m sglang.launch_server \
    --model $MODEL_PATH \
    --served-model-name glm-5-fp8 \
    --trust-remote-code \
    --disable-overlap-schedule \
    --enable-single-batch-overlap \
    --disable-cuda-graph \
    --tp-size 8 \
    --ep-size 8 \
    --dp-size 8 \
    --pp-size $NNODES \
    --enable-dp-lm-head \
    --moe-dense-tp-size 1 \
    --enable-dp-attention \
    --moe-a2a-backend deepep \
    --deepep-mode normal \
    --deepep-config "$DEEP_EP_CONFIG" \
    --attention-backend nsa \
    --nsa-prefill-backend $nsa_backend \
    --nsa-decode-backend $nsa_backend \
    --sampling-backend flashinfer \
    --mem-fraction-static 0.75 \
    --max-running-requests $((NNODES * 8)) \
    --chunked-prefill-size $((8 * 4096)) \
    --max-prefill-tokens $((128 * 1024)) \
    --dist-init-addr ${PREFILL_IP} \
    --nnodes ${NNODES} \
    --node-rank ${NODE_RANK} \
    --host 0.0.0.0 \
    --port ${SGLANG_PORT} \
    --log-requests-level 3 \
    --log-level info \
    --enable-metrics \
    --enable-metrics-for-all-schedulers \
    --export-metrics-to-file \
    --export-metrics-to-file-dir ${LOG_DIR} \
    --enable-cache-report \
    --load-balance-method round_robin \
    --disaggregation-mode prefill \
    --disaggregation-ib-device $SGLANG_IB_DEVICES > "${LOG_DIR}/P${NODE_RANK}_${HOST_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

Decode启动​

decoder启动脚本 decode_server.sh

#!/bin/bash
source /root/.local/bin/uv-virtualenvwrapper.sh
workon glm5


SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export PYTHONPATH=/sgl-workspace/sglang-glm5/python:$PYTHONPATH

export MATE_FORCE_JIT=1
export SGLANG_USE_MTT_F8GEMM=1
export SGLANG_USE_MTT_HGEMM=1
export SGLANG_USE_MTT_EPI_QKV=1
export SGLANG_USE_MTT_INDEXER=1
export SGLANG_NSA_FUSE_TOPK=1
export SGLANG_USE_MTT_ATTN=1

export TP_SOCKET_IFNAME=bond0
export GLOO_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
# export PYTORCH_MUSA_ALLOC_CONF=expandable_segments:True

# export MUSA_BLOCK_SCHEDULE_MODE=1
# export MUSA_USERQ=1
export MUSA_PRINT_ENV=0
export MUSA_LAUNCH_BLOCKING=0
# export MUSA_EXECUTION_TIMEOUT=12000
export MUSA_ERROR_DUMP_VERBOSE=1
export MUSA_ENABLE_LLC_OPT=1

export MCCL_PROTOS=2
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0

# mooncake
export MC_TE_METRIC=1
export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_SBO_COMBINE_SHARED=1 
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true

export SGLANG_DISAGGREGATION_HEARTBEAT_INTERVAL=1000
export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=300
export SGLANG_DISAGGREGATION_MAPPING_IB_DEVICE_TO_GPU=1
export SGLANG_HEALTH_CHECK_TIMEOUT=100

export SGLANG_TORCH_PROFILER_DIR=$SCRIPT_DIR/traces
export SGLANG_EXPERT_DISTRIBUTION_RECORDER_DIR=$SCRIPT_DIR/traces

nsa_backend="tilelang"
if [ $SGLANG_USE_MTT_ATTN -gt 0 ]; then
  nsa_backend="mtt_dsa"
fi

# 获取当前环境的 RDMA 设备名列表，在启动时作为 disaggregation-ib-device 参数值
IB_DEVS=()
             
for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        # 速率（如 200 Gb/sec）
        rate=$(cat "$port/rate" 2>/dev/null)
        # link_layer: InfiniBand / Ethernet (RoCE)
        link=$(cat "$port/link_layer" 2>/dev/null)
        # 状态为 Up
        state=$(cat "$port/state" 2>/dev/null)
        # 只要是 200G/400G 且状态为 Up（IB 或 RoCE 都收）
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]] && [[ "$state" == "4"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
# 排序 + 去重
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
# 生成 MCCL / SGLang 变量
MCCL_IB_HCA=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
SGLANG_IB_DEVICES="$MCCL_IB_HCA"
echo "IB_DEVICES=$MCCL_IB_HCA"

NODE_RANK=$1
DECODER_IP="${2}:5403"
LOG_DIR=$3
MODEL_PATH=$4
HOST_IP=$5
NNODES=$6

SGLANG_PORT=20143
graph_bs=4

mkdir -p "${LOG_DIR}"
rm -rf ~/.triton/cache/

nohup python3 -m sglang.launch_server \
    --model $MODEL_PATH \
    --trust-remote-code \
    --disable-overlap-schedule \
    --enable-single-batch-overlap \
    --cuda-graph-bs $(seq 1 $graph_bs) \
    --tp-size $((NNODES * 8)) \
    --ep-size $((NNODES * 8)) \
    --dp-size $((NNODES * 8)) \
    --enable-dp-lm-head \
    --moe-dense-tp-size 1 \
    --enable-dp-attention \
    --disable-shared-experts-fusion \
    --moe-a2a-backend deepep \
    --deepep-mode low_latency \
    --attention-backend nsa \
    --nsa-prefill-backend $nsa_backend \
    --nsa-decode-backend $nsa_backend \
    --sampling-backend flashinfer \
    --speculative-algorithm EAGLE \
    --speculative-num-steps 3 \
    --speculative-eagle-topk 1 \
    --speculative-num-draft-tokens 4 \
    --reasoning-parser glm45 \
    --tool-call-parser glm47  \
    --mem-fraction-static 0.75 \
    --max-running-requests $((NNODES * 8 * graph_bs)) \
    --dist-init-addr ${DECODER_IP} \
    --nnodes ${NNODES} \
    --node-rank ${NODE_RANK} \
    --host 0.0.0.0 \
    --port ${SGLANG_PORT} \
    --log-requests-level 3 \
    --log-level info \
    --enable-metrics \
    --enable-metrics-for-all-schedulers \
    --export-metrics-to-file \
    --export-metrics-to-file-dir ${LOG_DIR} \
    --prefill-round-robin-balance \
    --load-balance-method shortest_queue \
    --disaggregation-mode decode  \
    --disaggregation-ib-device $SGLANG_IB_DEVICES > "${LOG_DIR}/D${NODE_RANK}_${HOST_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

Router启动​

Router 启动脚本 router.sh

#!/bin/bash

source /root/.local/bin/uv-virtualenvwrapper.sh
workon glm5

export PYTHONPATH=/sgl-workspace/sglang-glm5/python:$PYTHONPATH

PREFILL_IP="http://${1}:20133"
DECODE_IP="http://$2:20143"
LOG_DIR=$3

ulimit -n 65535
nohup python -m sglang_router.launch_router \
    --pd-disaggregation \
    --prefill $PREFILL_IP \
    --decode $DECODE_IP \
    --host 0.0.0.0 \
    --request-timeout-secs 7200 \
    --port 30000 > "${LOG_DIR}/router_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

创建 hostfile 文件​

hostfile文件示例

192.168.100.101
192.168.100.102
192.168.100.103
192.168.100.104
192.168.100.105
192.168.100.106
192.168.100.107
192.168.100.108

一键启动所有服务​

一键运行脚本 run.sh

#!/bin/bash
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
#echo $CURRENT_TIME
#mkdir -p ./output/$CURRENT_TIME

PREFILL_SERVER_COUNT=${PREFILL_SERVER_COUNT:-4}
DECODER_SERVER_COUNT=${DECODER_SERVER_COUNT:-4}
WORLD_SIZE=$(( (PREFILL_SERVER_COUNT + DECODER_SERVER_COUNT) * 8 ))
port=62216

echo -e "\033[32mPREFILL_SERVER_COUNT: $PREFILL_SERVER_COUNT, DECODER_SERVER_COUNT: $DECODER_SERVER_COUNT, \033[0m"

set -u
  WORK_HOME="$PWD"
  EXPNAME="P${PREFILL_SERVER_COUNT}_D${DECODER_SERVER_COUNT}_gpus${WORLD_SIZE}"
  MODEL_PATH=${MODEL_PATH:-"/data/models/GLM-5-FP8"}
  HOSTFILE=./hostfile
  LOG_DIR=$WORK_HOME/output/$EXPNAME/$CURRENT_TIME/
  PREFILL_SCRIPT_FILE="$WORK_HOME/prefill_server.sh"
  DECODER_SCRIPT_FILE="$WORK_HOME/decode_server.sh"
  ROUTER_SCRIPT_FILE="$WORK_HOME/router.sh"
set +u

echo -e "\033[32mWORK_HOME: $WORK_HOME\033[0m"
echo -e "\033[32mMODEL_PATH: $MODEL_PATH\033[0m"
echo -e "\033[32mPREFILL_CRIPT_FILE: $PREFILL_SCRIPT_FILE\033[0m"
echo -e "\033[32mDECODER_SCRIPT_FILE: $DECODER_SCRIPT_FILE\033[0m"

# bash generate_hostfile.sh 
COUNT=0
hostlist=$(grep -v '^#\|^$' $HOSTFILE | awk '{print $1}' | xargs)

# Check if hostlist is empty
if [ -z "$hostlist" ]; then
  echo "Error: hostlist is empty. Please add IP addresses to the hostfile."
  exit 1
fi
PREFILL_MASTER_IP=$(head -n 1 $HOSTFILE)
DECODER_MASTER_IP=$(tail -n $DECODER_SERVER_COUNT $HOSTFILE | head -n1)
echo -e "\033[32mPREFILL_MASTER_IP: $PREFILL_MASTER_IP\033[0m"
echo -e "\033[32mDECODER_MASTER_IP: $DECODER_MASTER_IP\033[0m"


mkdir -p $LOG_DIR
for host in ${hostlist[@]}; do
  echo $host
  if [[ $COUNT -lt $PREFILL_SERVER_COUNT ]]; then
    PREFILL_INDEX=$COUNT
    cmd="bash -c 'bash $PREFILL_SCRIPT_FILE $PREFILL_INDEX $PREFILL_MASTER_IP $LOG_DIR $MODEL_PATH $host $PREFILL_SERVER_COUNT'"
    #cmd="bash -c 'cd $WORK_HOME; echo $PATH '"
    echo $cmd
    ssh -p $port -f -n $host $cmd
  else
    DECODER_INDEX=$((COUNT - PREFILL_SERVER_COUNT))
    cmd="bash -c 'bash $DECODER_SCRIPT_FILE $DECODER_INDEX $DECODER_MASTER_IP $LOG_DIR $MODEL_PATH $host $DECODER_SERVER_COUNT'"
    echo $cmd
    ssh -p $port -f -n $host $cmd
  fi

  ((COUNT++))
done
cmd="bash -c 'bash $ROUTER_SCRIPT_FILE $PREFILL_MASTER_IP $DECODER_MASTER_IP $LOG_DIR'"
echo $cmd
ssh -p $port -f -n $PREFILL_MASTER_IP $cmd
echo -e "Main log file: \033[34m$LOG_DIR\033[0m"

验证 API​

在运行 Router 的机器上（若 --net host，容器内访问即可），验证是否有正常输出

curl http://127.0.0.1:30000/v1/chat/completions  \
    -H "Content-Type: application/json"          \
    -d '{     
        "model": "glm-5-fp8",  
        "messages": [{"role": "user", "content": "你好，SGLang！"}],  
        "max_tokens": 100,      
        "temperature": 0.7    
  }'

预期返回 JSON，choices[0].message.content 为模型回复（示例与线上模型权重、路由配置有关）。

GLM-4.6/4.7 模型部署​

部署说明​

GLM-4.6/4.7 可以采用单机混布模式，也可以采用 1P1D 的PD分离部署模式。

# 启动单机混布模式
./launch_server.sh -m <模型路径> -mtp
# PD 分离部署模式，在两台机器上分别启动 prefill server 和 decode server
./launch_server.sh -m <模型路径> -mtp -pd p
./launch_server.sh -m <模型路径> -mtp -pd d

# 在启动 Prefill Server 的机器上启动 Router，注意更改脚本中对应的 IP
./router.sh

参数说明

  • 如果启动时不加mtp参数，则不使用多 Token 预测功能。
  • 模型路径中需要包含字符4.6、4.7。GLM 4.6和4.7用的tool call parser不同，脚本需要这个信息判断模型版本选择tool call parser。
  • page_size只支持64，所以 --page-size 参数不要更改。
  • max-prefill-tokens为128K，默认最大并发数为16，可按需修改。

脚本参考​

SGLang 服务启动启动脚本 launch_server.sh

#!/bin/bash
# SKIP_THIRD_PARTY=1 python setup_musa.py install
ep=2
disable_cuda_graph=0
disable_radix_cache=0
with_mtp=0
pd_mode=""
port=20133
model_path=""

source /root/.local/bin/uv-virtualenvwrapper.sh
workon glm4

export PYTHONPATH=/sgl-workspace/sglang-glm4/python:$PYTHONPATH

usage() {
    echo "用法: $0 [选项]"
    echo "选项:"
    echo "  -dg           禁用 CUDA Graph"
    echo "  -dc           禁用 Radix Cache"
    echo "  -mtp          使用mtp"
    echo "  -e <1|2|4|8>  设置 ep_size"
    echo "  -pd <p|d>     设置 PD 分离"
    echo "  -m <model_path>  设置模型路径"
    echo "  -h            显示此帮助信息"
    echo ""
    echo "示例:"
    echo "  $0 -dg -dc -e 2 -m /data/GLM-4.6-FP8  # 同时禁用两者并设置 ep_size=2, 也是默认行为"
    echo "  $0 -m /data/GLM-4.6-FP8 -mtp"
    echo "注意: 不支持单独使用 -g 或 -c 选项，请使用 -dg 或 -dc"
}

# 检查参数中是否包含无效的 -g 或 -c 选项
check_invalid_options() {
    for arg in "$@"; do
        if [ "$arg" = "-g" ] || [ "$arg" = "-c" ]; then
            echo "错误: 不支持单独使用 $arg 选项" >&2
            if [ "$arg" = "-g" ]; then
                echo "请使用 -dg 来禁用 CUDA Graph" >&2
            else
                echo "请使用 -dc 来禁用 Radix Cache" >&2
            fi
            echo ""
            usage
            exit 1
        fi
    done
}

check_invalid_options "$@"

# 注意：getopts不直接支持多字符选项，所以我们用特殊方法处理
while [[ $# -gt 0 ]]; do
    case "$1" in
    -dg)
        disable_cuda_graph=1
        shift
        ;;
    -dc)
        disable_radix_cache=1
        shift
        ;;
    -mtp)
        with_mtp=1
        shift
        ;;
    -e)
        ep=$2
        shift 2
        ;;
    -pd)
        pd_mode="$2"
        shift 2
        ;;
    -m)
        model_path="$2"
        shift 2
        ;;
    -h)
        usage
        exit 0
        ;;
    -*)
        echo "错误: 无效选项 $1" >&2
        usage
        exit 1
        ;;
    *)
        echo "错误: 不支持的参数 $1" >&2
        usage
        exit 1
        ;;
    esac
done

if [ $ep -eq 1 ]; then
    batch=8
    max_total_tokens=$((64 * 1024 * 5))
else
    batch=16
    max_total_tokens=$((64 * 1024 * 9 + 10240))
fi

echo "ep_size: $ep, batch: $batch, max_total_tokens: $max_total_tokens, model_path: $model_path, with_mtp: $with_mtp"

if [[ $model_path == "" ]]; then
    echo "需要设置 model_path"
    usage
    exit 1
elif [[ "$model_path" == *"4.6"* ]]; then
    served_model_name="glm-4.6-fp8"
    tool_call_parser="glm45"
elif [[ "$model_path" == *"4.7"* ]]; then
    served_model_name="glm-4.7-fp8"
    tool_call_parser="glm47"
else
    echo "警告：model_path中未找到匹配的版本号（4.6或4.7）"
    usage
    exit 1
fi
echo "served_model_name: $served_model_name, tool_call_parser: $tool_call_parser"

PYTHON_ARGS=()
if [ $disable_cuda_graph -gt 0 ]; then
    PYTHON_ARGS+=(--disable-cuda-graph)
fi

if [ $disable_radix_cache -gt 0 ]; then
    PYTHON_ARGS+=(--disable-radix-cache)
fi

if [ $with_mtp -gt 0 ]; then
    PYTHON_ARGS+=(--speculative-algorithm EAGLE)
    PYTHON_ARGS+=(--speculative-num-steps 1)
    PYTHON_ARGS+=(--speculative-eagle-topk 1)
    PYTHON_ARGS+=(--speculative-num-draft-tokens 2)
fi

SGLANG_IB_DEVICES=""
function ib_dev() {
    IB_DEVS=()
    for dev in /sys/class/infiniband/*; do
        ibdev=$(basename "$dev")

        for ib_port in "$dev"/ports/*; do
            # 速率（如 200 Gb/sec）
            rate=$(cat "$ib_port/rate" 2>/dev/null)
            # link_layer: InfiniBand / Ethernet (RoCE)
            link=$(cat "$ib_port/link_layer" 2>/dev/null)
            # 只要是 200G/400G（IB 或 RoCE 都收）
            if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]]; then
                IB_DEVS+=("$ibdev")
                break
            fi
        done
    done
    # 排序 + 去重
    IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
    # 生成 MCCL / SGLang 变量
    MCCL_IB_HCA=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
    SGLANG_IB_DEVICES=$MCCL_IB_HCA
}

if [[ $pd_mode == "p" ]]; then
    ib_dev
    PYTHON_ARGS+=(--disaggregation-mode prefill)
    PYTHON_ARGS+=(--disaggregation-ib-device ${SGLANG_IB_DEVICES})
    port=${PREFILL_PORT:-30000}
elif [[ $pd_mode == "d" ]]; then
    ib_dev
    PYTHON_ARGS+=(--disaggregation-mode decode)
    PYTHON_ARGS+=(--disaggregation-ib-device ${SGLANG_IB_DEVICES})
    port=${DECODE_PORT:-30000}
fi

echo "${PYTHON_ARGS[@]}"

export MATE_FORCE_JIT=1
export SGLANG_USE_MTT_GEMM=1
export SGLANG_USE_MTT_EPI_QKV=1
export SGLANG_USE_MTT_ATTN=1
export SGLANG_USE_MTT_MOE=1
export SGLANG_USE_MTT=1
export MUSA_LAUNCH_BLOCKING=0
export MUSA_BLOCK_SCHEDULE_MODE=1
export MUSA_USERQ=1

export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export MC_TE_METRIC=1
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=1
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export VLLM_CONFIGURE_LOGGING=0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
export NVSHMEM_IB_TRAFFIC_CLASS=136
export SGL_DISAGGREGATION_MAPPING_IB_DEVICE_TO_GPU=1
export MC_IB_TC=136
export MC_IB_PCIE_RELAXED_ORDERING=1

# export MUSA_PRINT_ENV=1
# export MUSA_LOG=0xffff
# export MUSA_EXECUTION_TIMEOUT=12000
# export PYTORCH_MUSA_ALLOC_CONF=expandable_segments:True

: <<'COMMENT'
    --disable-cuda-graph \
    --cuda-graph-max-bs $batch \
    --disable-shared-experts-fusion \
    --disable-radix-cache \
    --disaggregation-mode decode \
    --disaggregation-ib-device $SGLANG_IB_DEVICES \
    --skip-server-warmup \
    --enable-nan-detection \
COMMENT

python3 -m sglang.launch_server \
    --model-path $model_path \
    --served-model-name $served_model_name \
    "${PYTHON_ARGS[@]}" \
    --disable-overlap-schedule \
    --device musa \
    --tp-size 8 \
    --ep-size $ep \
    --page-size 64 \
    --mem-fraction-static 0.9 \
    --max-running-requests $batch \
    --max-total-tokens $max_total_tokens \
    --max-prefill-tokens $((64 * 1024 * 2)) \
    --chunked-prefill-size $((8192 * 1)) \
    --attention-backend fa3 \
    --sampling-backend flashinfer \
    --cuda-graph-bs $(seq 1 $batch) \
    --disable-cuda-graph-padding \
    --tool-call-parser $tool_call_parser \
    --reasoning-parser glm45 \
    --log-requests-level 3 \
    --enable-metrics \
    --enable-metrics-for-all-schedulers \
    --watchdog-timeout 100 \
    --host 0.0.0.0 \
    --port $port

Router 启动脚本 router.sh

#!/bin/bash
source /root/.local/bin/uv-virtualenvwrapper.sh
workon glm4

export PYTHONPATH=/sgl-workspace/sglang-glm4/python:$PYTHONPATH

# 注意更改脚本中对应的 IP
python -m sglang_router.launch_router \
    --pd-disaggregation \
    --prefill ${SGLANG_PREFILL_NODE:-http://192.168.100.101:20133} \
    --decode ${SGLANG_DECODE_NODE:-http://192.168.100.102:20133} \
    --host 0.0.0.0 \
    --port ${PORT:-30000}

版权所有 2024 摩尔线程，本文档受国际版权法保护。

摩尔线程和摩尔线程徽标是摩尔线程智能科技 (北京)有限责任公司的注册商标。

免责声明

本文档提供有关摩尔线程产品的信息。本文档并未授权任何知识产权的许可，并未以明示或暗示，或以禁止反言或其他方式授予任何知识产权许可。