Deepseek 模型部署

Deepseek-V4-Flash 模型部署

本章节包含 Deepseek-V4-Flash-FP8 的部署指导。

• Deepseek-V4-Flash-FP8 模型权重下载。

• 参考 环境准备 进行环境检查、启动容器。

• 容器镜像地址参考 版本发布信息-镜像地址。

• 脚本涉及的参数说明和一键运行说明见 快速开始。

• 暂不支持 overlap-schedule 和 EPLB，会引发精度和稳定性问题.

PD 分离部署

Prefill 节点负责预填充，Decode 节点负责解码。二者通过 Mooncake 与 InfiniBand 传输 KV，最前端的 Router 统一对外暴露 HTTP 接口。

Client → Router (:8192) → Prefill (:23456) / Decode (:23457)

PD 分离部署的最小单元是 prefill 和 Decode 实例各用1台机器。可根据上下文长度和性能要求，适当增加 Prefill 实例的数量，使 Prefill 和 Decode 的吞吐相匹配。建议如下：

输入长度	拓扑
32k以下	1P1D
32k	3P1D
64k	4P1D
96k	3P1D
128k	3P1D

Prefill启动

prefill启动脚本 start_prefill.sh

#!/bin/bash
# DeepSeek V4 MUSA PD Disaggregation - Prefill Node (统一启动脚本)
#
# Usage:
#   bash start_prefill.sh [MODEL_PATH]
#
# Example:
#   bash start_prefill.sh
#   bash start_prefill.sh /data/models/DeepSeek-V4-Flash-FP8

set -euo pipefail

# 自动获取本机 IP（优先走默认路由的网卡地址）
get_host_ip() {
    ip -4 route get 1.1.1.1 2>/dev/null | awk '{for (i = 1; i <= NF; i++) if ($i == "src") { print $(i + 1); exit }}'
}
HOST_IP=$(get_host_ip)
if [[ -z "${HOST_IP}" ]]; then
    HOST_IP=$(hostname -I | awk '{print $1}')
fi
if [[ -z "${HOST_IP}" ]]; then
    echo "ERROR: 无法自动获取本机 IP" >&2
    exit 1
fi

MODEL_PATH="${1:-/data/models/DeepSeek-V4-Flash-FP8}"
PREFILL_IP="${HOST_IP}:29500"

DEEP_EP_CONFIG='{"normal_dispatch":{"num_sms":60,"num_max_nvl_chunked_send_tokens":26},"normal_combine":{"num_sms":60,"num_max_nvl_chunked_send_tokens":16}}'

echo "HOST_IP=${HOST_IP}  MODEL_PATH=${MODEL_PATH}"

# IB 设备自动检测（200G / 400G）
IB_DEVS=()
for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        rate=$(cat "$port/rate" 2>/dev/null)
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
SGLANG_IB_DEVICES=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
export MCCL_IB_HCA="$SGLANG_IB_DEVICES"

export SGLANG_DSV4_FP4_EXPERTS=0
export SGLANG_OPT_USE_FUSED_HASH_TOPK=1
export SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK=1
export SGLANG_JIT_DEEPGEMM_PRECOMPILE=0
export SGLANG_DSV4_MUSA_MOE_EXPERIMENTAL=1
export SGLANG_OPT_USE_JIT_INDEXER_METADATA=1
export SGLANG_OPT_USE_JIT_NORM=0
export SGLANG_OPT_FLASHMLA_SPARSE_PREFILL=0
export SGLANG_DEEP_GEMM_BLOCK_M=256

# MHC
export SGLANG_OPT_USE_TILELANG_MHC_PRE=1
export SGLANG_OPT_MHC_PRE_BACKEND=auto
export SGLANG_OPT_DEEPGEMM_HC_PRENORM=1
export SGLANG_OPT_DEEPGEMM_HC_PRENORM_SPLIT_K=32
export SGLANG_OPT_MHC_PRE_BIG_FUSE_THREADS=0
export SGLANG_OPT_MHC_PRE_BIG_FUSE_HIDDEN_BLOCK=0
export SGLANG_OPT_MHC_PRE_BIG_FUSE_PASS_CONFIG=auto
export SGLANG_OPT_MHC_PRENORM_BACKEND=deepgemm
export SGLANG_OPT_MHC_PRENORM_SPLIT_K=32
export SGLANG_OPT_MHC_PRENORM_TILELANG_IMPL=auto
export SGLANG_OPT_USE_TILELANG_MHC_POST=1
export SGLANG_OPT_MHC_POST_THREADS=0
export SGLANG_OPT_MHC_POST_HIDDEN_BLOCK=0
export SGLANG_OPT_MHC_POST_PASS_CONFIG=auto
export SGLANG_OPT_MHC_POST_LAYOUT=auto
export SGLANG_OPT_MHC_POST_DIRECT_STORE=0

# GEMM / HC head
export SGLANG_OPT_BF16_FP32_GEMM_ALGO=deep_gemm
export SGLANG_OPT_HC_HEAD_TILELANG=1

# TopK
export SGLANG_DEEPSEEK_V4_MUSA_ENABLE_JIT_TOPK512=1

# RoPE / Hadamard
export SGLANG_OPT_DSV4_MUSA_TILELANG_ROPE=0
export SGLANG_OPT_DSV4_MUSA_FUSED_ROPE_HADAMARD=1
export SGLANG_OPT_DSV4_MUSA_FUSED_NSA_ROPE_HADAMARD=1
export SGLANG_OPT_DSV4_MUSA_TILELANG_WO_A=0

# Quant
export SGLANG_OPT_USE_TILEKERNELS_FP8_QUANT=1
export SGLANG_OPT_USE_TILEKERNELS_SWIGLU_QUANT=1

# MTP
export SGLANG_PREP_IN_CUDA_GRAPH=0
export SGLANG_DSV4_EAGLE_DRAFT_EXTEND_CUDA_GRAPH=0
export SGLANG_DSV4_HC_HEAD_TP=0

# KV transfer
export SGLANG_DISAGGREGATION_QUEUE_SIZE=32
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=64

# c128 online
export SGLANG_DSV4_COMPRESS_STATE_DTYPE=bf16
export SGLANG_OPT_USE_ONLINE_COMPRESS=1
export SGLANG_OPT_USE_COMPRESSOR_V2_MUSA=1
export SGLANG_DISAGGREGATION_ALL_CP_RANKS_TRANSFER=1

export SGLANG_HEALTH_CHECK_TIMEOUT=1200

python -m sglang.launch_server \
    --trust-remote-code \
    --model-path "$MODEL_PATH" \
    --tp 8 \
    --ep 8 \
    --mem-fraction-static 0.78 \
    --swa-full-tokens-ratio 0.2 \
    --host 0.0.0.0 \
    --port 23456 \
    --chunked-prefill-size 40960 \
    --max-prefill-tokens 65536 \
    --disable-overlap-schedule \
    --log-level info \
    --watchdog-timeout 6000 \
    --disable-cuda-graph \
    --enable-nsa-prefill-context-parallel \
    --nsa-prefill-cp-mode round-robin-split \
    --moe-a2a-backend deepep \
    --deepep-mode normal \
    --deepep-config "$DEEP_EP_CONFIG" \
    --disaggregation-mode prefill \
    --disaggregation-transfer-backend mooncake \
    --disaggregation-bootstrap-port 8998 \
    --disaggregation-ib-device "${SGLANG_IB_DEVICES}" \
    --reasoning-parser deepseek-v4 \
    --tool-call-parser deepseekv4 \
    --enable-metrics \
    --dist-init-addr "$PREFILL_IP"

---

Decode启动

decode启动脚本 start_decode.sh

#!/bin/bash
# DeepSeek V4 MUSA PD Disaggregation - Decode Node (Attn DP8 + MoE TP8)
#
# Usage:
#   bash start_decode.sh [MODEL_PATH]
#
# Example:
#   bash start_decode.sh
#   bash start_decode.sh /data/models/DeepSeek-V4-Flash-FP8

set -euo pipefail

# 自动获取本机 IP（优先走默认路由的网卡地址）
get_host_ip() {
    ip -4 route get 1.1.1.1 2>/dev/null | awk '{for (i = 1; i <= NF; i++) if ($i == "src") { print $(i + 1); exit }}'
}
HOST_IP=$(get_host_ip)
if [[ -z "${HOST_IP}" ]]; then
    HOST_IP=$(hostname -I | awk '{print $1}')
fi
if [[ -z "${HOST_IP}" ]]; then
    echo "ERROR: 无法自动获取本机 IP" >&2
    exit 1
fi

MODEL_PATH="${1:-/data/models/DeepSeek-V4-Flash-FP8}"
DECODER_IP="${HOST_IP}:29500"

echo "HOST_IP=${HOST_IP}  MODEL_PATH=${MODEL_PATH}"

# IB 设备自动检测（200G / 400G）
IB_DEVS=()
for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        rate=$(cat "$port/rate" 2>/dev/null)
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
SGLANG_IB_DEVICES=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
export MCCL_IB_HCA="$SGLANG_IB_DEVICES"

export SGLANG_DSV4_FP4_EXPERTS=0
export SGLANG_OPT_USE_FUSED_HASH_TOPK=1
export SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK=1
export SGLANG_JIT_DEEPGEMM_PRECOMPILE=1
export SGLANG_OPT_USE_JIT_EP_ACTIVATION=1
export SGLANG_DEEP_GEMM_BLOCK_M=256

# MHC
export SGLANG_OPT_USE_TILELANG_MHC_PRE=1
export SGLANG_OPT_MHC_PRE_BACKEND=auto
export SGLANG_OPT_DEEPGEMM_HC_PRENORM=1
export SGLANG_OPT_DEEPGEMM_HC_PRENORM_SPLIT_K=32
export SGLANG_OPT_MHC_PRE_BIG_FUSE_THREADS=0
export SGLANG_OPT_MHC_PRE_BIG_FUSE_HIDDEN_BLOCK=0
export SGLANG_OPT_MHC_PRE_BIG_FUSE_PASS_CONFIG=auto
export SGLANG_OPT_MHC_PRENORM_BACKEND=deepgemm
export SGLANG_OPT_MHC_PRENORM_SPLIT_K=32
export SGLANG_OPT_MHC_PRENORM_TILELANG_IMPL=auto
export SGLANG_OPT_USE_TILELANG_MHC_POST=1
export SGLANG_OPT_MHC_POST_THREADS=0
export SGLANG_OPT_MHC_POST_HIDDEN_BLOCK=0
export SGLANG_OPT_MHC_POST_PASS_CONFIG=auto
export SGLANG_OPT_MHC_POST_LAYOUT=auto
export SGLANG_OPT_MHC_POST_DIRECT_STORE=0

# GEMM / HC head
export SGLANG_OPT_BF16_FP32_GEMM_ALGO=deep_gemm
export SGLANG_OPT_HC_HEAD_TILELANG=0

# TopK
export SGLANG_DEEPSEEK_V4_MUSA_ENABLE_JIT_TOPK512=1

# RoPE / Hadamard
export SGLANG_OPT_DSV4_MUSA_TILELANG_ROPE=0
export SGLANG_OPT_DSV4_MUSA_FUSED_ROPE_HADAMARD=1
export SGLANG_OPT_DSV4_MUSA_FUSED_NSA_ROPE_HADAMARD=1
export SGLANG_OPT_DSV4_MUSA_TILELANG_WO_A=0

# Quant
export SGLANG_OPT_USE_TILEKERNELS_FP8_QUANT=1
export SGLANG_OPT_USE_TILEKERNELS_SWIGLU_QUANT=1

# MTP / EAGLE graph preparation optimizations.
export SGLANG_PREP_IN_CUDA_GRAPH=1
export SGLANG_DSV4_EAGLE_DRAFT_EXTEND_CUDA_GRAPH=1
export SGLANG_DSV4_HC_HEAD_TP=0

# KV transfer
export SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS=128

# c128 online
export SGLANG_DSV4_COMPRESS_STATE_DTYPE=bf16
export SGLANG_OPT_USE_ONLINE_COMPRESS=1
export SGLANG_OPT_USE_COMPRESSOR_V2_MUSA=1
export SGLANG_DISAGGREGATION_ALL_CP_RANKS_TRANSFER=1

python -m sglang.launch_server \
    --trust-remote-code \
    --model-path "$MODEL_PATH" \
    --tp 8 \
    --max-running-requests 144 \
    --num-reserved-decode-tokens 256 \
    --disaggregation-mode decode \
    --disaggregation-transfer-backend mooncake \
    --disaggregation-bootstrap-port 8998 \
    --disaggregation-ib-device "${SGLANG_IB_DEVICES}" \
    --dist-init-addr "$DECODER_IP" \
    --mem-fraction-static 0.9 \
    --swa-full-tokens-ratio 0.35 \
    --host 0.0.0.0 \
    --port 23457 \
    --cuda-graph-max-bs 128 \
    --chunked-prefill-size 8192 \
    --disable-overlap-schedule \
    --skip-server-warmup \
    --speculative-algo EAGLE \
    --speculative-num-steps 3 \
    --speculative-eagle-topk 1 \
    --speculative-num-draft-tokens 4 \
    --dp 8 \
    --enable-dp-attention \
    --reasoning-parser deepseek-v4 \
    --tool-call-parser deepseekv4 \
    --enable-metrics

---

参考启动命令

以下示例命令启动 3个 Prefill 实例，1个 Decode 实例。如果只需要1个 Prefill 实例，则只需启动 prefill node 0，启动 Router 节点也只需第一个 --prefill 参数。

Prefill 节点：

# prefill node 0 执行
bash start_prefill.sh /data/models/DeepSeek-V4-Flash-FP8
# prefill node 1 执行
bash start_prefill.sh /data/models/DeepSeek-V4-Flash-FP8
# prefill node 2 执行
bash start_prefill.sh /data/models/DeepSeek-V4-Flash-FP8

Decode 节点：

# decode node 执行
bash start_decode.sh /data/models/DeepSeek-V4-Flash-FP8

Router 节点：

# prefill node 0 上执行
python3 -m sglang_router.launch_router   \
  --pd-disaggregation \
  --prefill http://<prefill_0_ip>:23456 8998 \
  --prefill http://<prefill_1_ip>:23456 8998 \
  --prefill http://<prefill_2_ip>:23456 8998 \
  --decode http://<decode_0_ip>:23457 \
  --host 0.0.0.0 \
  --port 8192 \
  --prefill-policy round_robin \
  --decode-policy round_robin \
  --worker-startup-timeout-secs 1200 \
  --health-check-timeout-secs 1200 \
  --prometheus-port 29001

---

单机部署

服务启动脚本参考

服务启动脚本 start_server.sh

#!/bin/bash
# DeepSeek V4 MUSA PD Disaggregation - Prefill Node

# IB 设备自动检测（检测 200G/400G IB 设备）
IB_DEVS=()
for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        rate=$(cat "$port/rate" 2>/dev/null)
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
SGLANG_IB_DEVICES=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
MCCL_IB_HCA="$SGLANG_IB_DEVICES"
export MCCL_IB_HCA

MODEL_PATH=$1

# 必要的环境变量（workaround 和功能配置）
export SGLANG_DSV4_FP4_EXPERTS=0
export SGLANG_ENABLE_JIT_DEEPGEMM=1
export SGLANG_OPT_USE_FUSED_HASH_TOPK=1
export SGLANG_JIT_DEEPGEMM_PRECOMPILE=0
export SGLANG_DSV4_MUSA_MOE_EXPERIMENTAL=1
export SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK=1
export SGLANG_OPT_USE_JIT_INDEXER_METADATA=1
export SGLANG_OPT_USE_JIT_NORM=0
export SGLANG_OPT_FLASHMLA_SPARSE_PREFILL=0

# MHC
export SGLANG_OPT_USE_TILELANG_MHC_PRE=1
export SGLANG_OPT_MHC_PRE_BACKEND=auto
export SGLANG_OPT_DEEPGEMM_HC_PRENORM=1
export SGLANG_OPT_DEEPGEMM_HC_PRENORM_SPLIT_K=32
export SGLANG_OPT_MHC_PRE_BIG_FUSE_THREADS=0
export SGLANG_OPT_MHC_PRE_BIG_FUSE_HIDDEN_BLOCK=0
export SGLANG_OPT_MHC_PRE_BIG_FUSE_PASS_CONFIG=auto
export SGLANG_OPT_MHC_PRENORM_BACKEND=deepgemm
export SGLANG_OPT_MHC_PRENORM_SPLIT_K=32
export SGLANG_OPT_MHC_PRENORM_TILELANG_IMPL=auto
export SGLANG_OPT_USE_TILELANG_MHC_POST=1
export SGLANG_OPT_MHC_POST_THREADS=0
export SGLANG_OPT_MHC_POST_HIDDEN_BLOCK=0
export SGLANG_OPT_MHC_POST_PASS_CONFIG=auto
export SGLANG_OPT_MHC_POST_LAYOUT=auto
export SGLANG_OPT_MHC_POST_DIRECT_STORE=0

# GEMM / HC head
export SGLANG_OPT_BF16_FP32_GEMM_ALGO=deep_gemm
export SGLANG_OPT_HC_HEAD_TILELANG=1
export SGLANG_DEEP_GEMM_BLOCK_M=256


# TopK
export SGLANG_DEEPSEEK_V4_MUSA_ENABLE_JIT_TOPK512=1

# RoPE / Hadamard
export SGLANG_OPT_DSV4_MUSA_TILELANG_ROPE=0
export SGLANG_OPT_DSV4_MUSA_FUSED_ROPE_HADAMARD=1
export SGLANG_OPT_DSV4_MUSA_FUSED_NSA_ROPE_HADAMARD=1

# Quant
export SGLANG_OPT_USE_TILEKERNELS_FP8_QUANT=1
export SGLANG_OPT_USE_TILEKERNELS_SWIGLU_QUANT=1

#MTP
export SGLANG_PREP_IN_CUDA_GRAPH=0
export SGLANG_DSV4_EAGLE_DRAFT_EXTEND_CUDA_GRAPH=0
export SGLANG_DSV4_HC_HEAD_TP=0

export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600

python -m sglang.launch_server \
    --trust-remote-code \
    --model-path "$MODEL_PATH" \
    --tp 8 \
    --ep 8 \
    --mem-fraction-static 0.8 \
    --host 0.0.0.0 \
    --port 24586 \
    --chunked-prefill-size 40960 \
    --max-prefill-tokens 60000 \
    --disable-overlap-schedule \
    --disable-custom-all-reduce \
    --log-level info \
    --tokenizer-backend fastokens \
    --watchdog-timeout 6000 \
    --enable-nsa-prefill-context-parallel \
    --nsa-prefill-cp-mode round-robin-split \
    --moe-a2a-backend deepep \
    --deepep-mode auto \
    --reasoning-parser deepseek-v4 \
    --tool-call-parser deepseekv4 \
    --cuda-graph-max-bs 16 

启动命令参考

bash start_server.sh /data/models/DeepSeek-V4-Flash-FP8

---

Deepseek V3.2 模型部署

本节包含 Deepseek V3.2 部署脚本。

• 模型权重下载。

• 参考 环境准备 进行环境检查、启动容器。

• 容器镜像地址和Qwen系列相同，参考 版本发布信息-镜像地址。

• 脚本涉及的参数说明和一键运行说明见 快速开始。

• DeepEP 的配置文件内容参考 DeepEP 配置文件

该示例启动方式为2P4D，请准备6台机器。最小部署方式为2机混布（如 TP8 PP2）。

prefill启动脚本 prefill_server.sh

#!/bin/bash
export SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_IDLE=0
export TP_SOCKET_IFNAME=bond1
export GLOO_SOCKET_IFNAME=bond1
export MUSA_LAUNCH_BLOCKING=0
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
# export SGLANG_USE_DEEPGEMM_BMM=0
export MC_TE_METRIC=true
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_PP_LAYER_PARTITION="32,29"
#export SGLANG_PP_LAYER_PARTITION="31,30"

export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=false

export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export SGLANG_MOE_PADDING=1

#export LD_LIBRARY_PATH=/usr/local/musa/lib:$LD_LIBRARY_PATH
# export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH}
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:/usr/local/musa/lib:$LD_LIBRARY_PATH


export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_USE_DEEPGEMM_BMM=0
export SGLANG_MOE_PADDING=1
export SGLANG_USE_MUSA_FUSED_KERNEL=1
export VLLM_CONFIGURE_LOGGING=0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
export SGLANG_DEEPEP_BF16_DISPATCH=0

export NVSHMEM_IB_TRAFFIC_CLASS=136
#export MC_IB_TC=136

#export SGLANG_USE_CUDA_ROTARY_EMBEDDING=1

# export SGLANG_USE_MTT=1

#export MC_ENABLE_DEST_DEVICE_AFFINITY=1
export SGLANG_DISAGGREGATION_MAPPING_IB_DEVICE_TO_GPU=1
export SGLANG_DEEP_GEMM_BLOCK_M=256

export MC_IB_TC=136
export MC_IB_PCIE_RELAXED_ORDERING=1

NODE_RANK=$1
PREFILL_IP="${2}:2345"
LOG_DIR=$3
MODEL_PATH=$4
HOST_IP=$5

PROFILER_DIR_BASE="$PWD"
export SGLANG_TORCH_PROFILER_DIR="${PROFILER_DIR_BASE}/traces/sglang_prefill_profiling_p${NODE_RANK}"

mkdir -p "${LOG_DIR}"

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

DEEP_EP_CONFIG="${SCRIPT_DIR}/deepep.config"

export SGLANG_EXPERT_DISTRIBUTION_RECORDER_DIR=/data/workspace/dsv32

export LD_LIBRARY_PATH=/usr/local/lib/:/usr/local/mtshmem/lib/:/usr/local/musa/lib:/usr/local/openmpi/lib:/usr/local/musa/mudnn/lib:/usr/lib/x86_64-linux-gnu:ldconfig


#echo -e "\033[32mLOG_DIR: ${LOG_DIR}\033[0m"
nohup python3 -m sglang.launch_server \
            --model-path $MODEL_PATH \
            --served-model-name base-model \
            --trust-remote-code \
            --disable-cuda-graph \
            --disable-overlap-schedule \
            --host 0.0.0.0 \
            --port 24588 \
            --attention-backend nsa \
            --nsa-prefill-backend tilelang \
            --nsa-decode-backend tilelang \
            --sampling-backend flashinfer \
            --disaggregation-mode prefill \
            --dist-init-addr $PREFILL_IP \
            --chunked-prefill-size -1 \
            --max-prefill-tokens 4096 \
            --nnodes 2  \
            --node-rank $NODE_RANK \
            --tensor-parallel-size 8 \
            --ep-size 8 \
            --dp-size 8 \
            --context-length 6000 \
            --pp-size 2 \
            --moe-a2a-backend deepep \
            --deepep-mode normal \
            --moe-dense-tp-size 1 \
            --enable-dp-attention \
            --enable-dp-lm-head \
            --load-balance-method round_robin \
            --deepep-config "$DEEP_EP_CONFIG" \
            --max-running-requests 64 \
            --enable-cache-report \
            --mem-fraction-static 0.8 \
            --schedule-conservativeness 1 \
	        --disaggregation-ib-device mlx5_bond_2,mlx5_bond_2,mlx5_bond_3,mlx5_bond_3,mlx5_bond_4,mlx5_bond_4,mlx5_bond_5,mlx5_bond_5 \
            --disable-shared-experts-fusion > "${LOG_DIR}/P${NODE_RANK}_${HOST_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

decoder启动脚本 decode_server.sh

export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export MUSA_LAUNCH_BLOCKING=0
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
#export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16
export SGLANG_USE_DEEPGEMM_BMM=0
export MC_TE_METRIC=true
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=600
export SGLANG_TORCH_PROFILER_DIR=/data/workspace/dsv32

export NVSHMEM_IB_ENABLE_SHUFFLE=0 
export NVSHMEM_ENABLE_NIC_PE_MAPPING=1 
export NVSHMEM_HCA_PE_MAPPING=mlx5_bond_2:1:2,mlx5_bond_3:1:2,mlx5_bond_4:1:2,mlx5_bond_5:1:2
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=false
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export SGLANG_MOE_PADDING=1

export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_MOE_PADDING=1
export SGLANG_USE_MUSA_FUSED_KERNEL=1
export VLLM_CONFIGURE_LOGGING=0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export SGLANG_DEEPEP_LOW_LATENCY_USE_NVLINK=1
#export SGLANG_USE_CUDA_ROTARY_EMBEDDING=1

#export LD_LIBRARY_PATH=/usr/local/musa/lib:$LD_LIBRARY_PATH
# export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH}
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:/usr/local/musa/lib:$LD_LIBRARY_PATH
export SGLANG_DEEPEP_BF16_DISPATCH=0

export MC_IB_TC=136
export MC_IB_PCIE_RELAXED_ORDERING=1

export SGLANG_USE_MTT=0

export NVSHMEM_IB_TRAFFIC_CLASS=136

#export MC_ENABLE_DEST_DEVICE_AFFINITY=1
# export SGL_DISAGGREGATION_MAPPING_IB_DEVICE_TO_GPU=1
export SGLANG_DISAGGREGATION_MAPPING_IB_DEVICE_TO_GPU=1
export MUSA_DEVICE_PAGE_SIZE=0x1000
#export MUSA_MANAGED_FORCE_DEVICE_ALLOC=1
export NVSHMEM_IBGDA_NIC_HANDLER=gpu
export NVSHMEM_IBGDA_FORCE_NIC_BUF_MEMTYPE=gpumem

NODE_RANK=$1
DECODER_IP="${2}:4343"
LOG_DIR=$3
MODEL_PATH=$4
HOST_IP=$5

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
export LD_LIBRARY_PATH=/usr/local/lib/:/usr/local/mtshmem/lib/:/usr/local/musa/lib:/usr/local/musa/lib:/usr/local/openmpi/lib:/usr/local/musa/mudnn/lib:/usr/lib/x86_64-linux-gnu:ldconfig
nohup python3 -m sglang.launch_server \
        --model $MODEL_PATH \
        --trust-remote-code \
        --ep-size 32 \
        --tp-size 32 \
        --dp-size 32 \
        --cuda-graph-bs $(seq 1 16) \
        --mem-fraction-static 0.76 \
        --speculative-algorithm EAGLE \
        --speculative-num-steps 3 \
        --speculative-eagle-topk 1 \
        --speculative-num-draft-tokens 4 \
        --enable-dp-lm-head \
        --enable-dp-attention \
        --port 23457 \
        --moe-a2a-backend deepep \
        --deepep-mode low_latency \
        --dist-init-addr $DECODER_IP \
        --nnodes 4 \
        --node-rank $NODE_RANK \
        --max-running-requests 1920 \
        --attention-backend nsa \
        --nsa-prefill-backend tilelang \
        --nsa-decode-backend tilelang \
        --moe-dense-tp-size 1 \
        --host 0.0.0.0 \
        --sampling-backend flashinfer \
        --disable-shared-experts-fusion \
        --chunked-prefill-size -1 \
        --load-balance-method round_robin \
        --decode-log-interval 1 \
        --enable-single-batch-overlap \
        --disaggregation-ib-device mlx5_bond_2,mlx5_bond_2,mlx5_bond_3,mlx5_bond_3,mlx5_bond_4,mlx5_bond_4,mlx5_bond_5,mlx5_bond_5 \
        --disaggregation-mode decode \
        --schedule-conservativeness 0.8 \
--tool-call-parser deepseekv32 \
        --reasoning-parser deepseek-v3 \
        --prefill-round-robin-balance > "${LOG_DIR}/D${NODE_RANK}_${HOST_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

# --context-length 16000 \

Router 启动脚本 router.sh

export PATH=/root/.virtualenvs/sglang-default/bin/:$PATH

PREFILL_IP="http://${1}:24588"
DECODE_IP="http://$2:23457"
LOG_DIR=$3

ulimit -n 65535
nohup python -m sglang_router.launch_router \
    --pd-disaggregation \
    --mini-lb \
    --prefill $PREFILL_IP \
    --decode $DECODE_IP \
    --host 0.0.0.0 \
    --request-timeout-secs 7200 \
    --port 22230 > "${LOG_DIR}/router_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

一键运行脚本（参考）。将Prefill启动脚本，Decode启动脚本，Router启动脚本以及run.sh放在同一路径下。并将6台节点的IP放进hostfile文件中。

bash run.sh

一键运行脚本 run.sh

#!/bin/bash

CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
#echo $CURRENT_TIME
#mkdir -p ./output/$CURRENT_TIME

PREFILL_SERVER_COUNT=${PREFILL_SERVER_COUNT:-2}
DECODER_SERVER_COUNT=${DECODER_SERVER_COUNT:-4}
WORLD_SIZE=$(( (PREFILL_SERVER_COUNT + DECODER_SERVER_COUNT) * 8 ))

echo -e "\033[32mPREFILL_SERVER_COUNT: $PREFILL_SERVER_COUNT, DECODER_SERVER_COUNT: $DECODER_SERVER_COUNT, \033[0m"

set -u
  WORK_HOME="$PWD"
  EXPNAME="P${PREFILL_SERVER_COUNT}_D${DECODER_SERVER_COUNT}_gpus${WORLD_SIZE}"
  MODEL_PATH=${MODEL_PATH:-"/data/models/DeepSeek-V3.2"}
  HOSTFILE=./hostfile
  LOG_DIR=$WORK_HOME/output/$EXPNAME/$CURRENT_TIME/
  PREFILL_SCRIPT_FILE="$WORK_HOME/prefill_server.sh"
  DECODER_SCRIPT_FILE="$WORK_HOME/decode_server.sh"
  ROUTER_SCRIPT_FILE="$WORK_HOME/router.sh"
  #ROUTER_SCRIPT_FILE="$WORK_HOME/router_sgl.sh"
set +u

echo -e "\033[32mWORK_HOME: $WORK_HOME\033[0m"
echo -e "\033[32mMODEL_PATH: $MODEL_PATH\033[0m"
echo -e "\033[32mPREFILL_CRIPT_FILE: $PREFILL_SCRIPT_FILE\033[0m"
echo -e "\033[32mDECODER_SCRIPT_FILE: $DECODER_SCRIPT_FILE\033[0m"

# bash generate_hostfile.sh 
COUNT=0
hostlist=$(grep -v '^#\|^$' $HOSTFILE | awk '{print $1}' | xargs)

# Check if hostlist is empty
if [ -z "$hostlist" ]; then
  echo "Error: hostlist is empty. Please add IP addresses to the hostfile."
  exit 1
fi

PREFILL_MASTER_IP=$(head -n 1 $HOSTFILE)
DECODER_MASTER_IP=$(tail -n $DECODER_SERVER_COUNT $HOSTFILE | head -n1)
echo -e "\033[32mPREFILL_MASTER_IP: $PREFILL_MASTER_IP\033[0m"
echo -e "\033[32mDECODER_MASTER_IP: $DECODER_MASTER_IP\033[0m"


mkdir -p $LOG_DIR
for host in ${hostlist[@]}; do
  echo $host
  if [[ $COUNT -lt $PREFILL_SERVER_COUNT ]]; then
    PREFILL_INDEX=$COUNT
    cmd="bash -c 'cd $WORK_HOME;source ~/.virtualenvs/sglang-default/bin/activate; bash $PREFILL_SCRIPT_FILE $PREFILL_INDEX $PREFILL_MASTER_IP $LOG_DIR $MODEL_PATH $host'"
    #cmd="bash -c 'cd $WORK_HOME; echo $PATH '"
    echo $cmd
    ssh -p 62216 -f -n $host $cmd
  else
    DECODER_INDEX=$((COUNT - PREFILL_SERVER_COUNT))
    cmd="bash -c 'cd $WORK_HOME;source ~/.virtualenvs/sglang-default/bin/activate; bash $DECODER_SCRIPT_FILE $DECODER_INDEX $DECODER_MASTER_IP $LOG_DIR $MODEL_PATH $host'"
    echo $cmd
    ssh -p 62216 -f -n $host $cmd
  fi
  

#  cmd_ssh=$cmd" > $LOG_FILE.$COUNT.$host 2>&1'"

#  echo $cmd_ssh

#  if [[ "$RUN_LOCAL" -ne 0 ]]; then
#    eval $cmd_ssh
#  else
#  ssh -p 62216 -n $host $cmd_ssh
#  fi

#  # echo $host, "bash -c 'cd $FlagScale_HOME/megatron; nohup bash $SCRIPT_FILE $PROJ_HOME $EXPNAME $HOSTFILE \"$DATA_PATH\" >> $LOG_FILE.$COUNT.$host 2>&1 &'"
#  # ssh -f -n $host "bash -c 'cd $FlagScale_HOME/megatron; nohup bash $SCRIPT_FILE $PROJ_HOME $EXPNAME $HOSTFILE \"$DATA_PATH\" >> $LOG_FILE.$COUNT.$host 2>&1 &'"
  ((COUNT++))
done
cmd="bash -c 'cd $WORK_HOME;source ~/.virtualenvs/sglang-default/bin/activate; bash $ROUTER_SCRIPT_FILE $PREFILL_MASTER_IP $DECODER_MASTER_IP $LOG_DIR'"
echo $cmd
ssh -p 62216 -f -n $PREFILL_MASTER_IP $cmd
echo -e "Main log file: \033[34m$LOG_DIR\033[0m"