Deepseek 模型部署

Deepseek-V4-Flash-0731 模型部署

权重与镜像

Deepseek-V4-Flash-0731 是 DeepSeek-V4-Flash 官方正式版本，取代了预览版。目前 SGLang-MUSA 不支持 BF16 权重，只支持 FP8 量化后的权重。请从魔搭社区摩尔线程组织页面下载模型权重。

本节内容适用于以下容器镜像地址，部署前参考 环境准备 进行环境检查、启动容器：

CPU 平台	镜像地址
Intel/AMD	registry.mthreads.com/mcconline/inference/sglang:v0.5.13-dsv4-5.0-amd64-ph1-musa4.3.8
Hygon	registry.mthreads.com/mcconline/inference/sglang:v0.5.13-dsv4-5.0-hygon-ph1-musa4.3.8

上述 dsv4-5.0 镜像相比 dsv4-3.1 镜像增加以下功能，注意暂不支持 overlap-schedule 和 EPLB，会引发精度和稳定性问题：

• speculative-dspark-block-size 支持 5。

• 支持 L2 和 L3 的 HiCache。

• decode kvcache 支持自动 cpu offload。

• prefill支持 mqa_logits 分块计算降低显存压力，设置环境变量 SGLANG_DSV4_MQA_LOGITS_FREE_MEM_FRACTION。

PD 分离部署

Prefill 节点负责预填充，Decode 节点负责解码。二者通过 Mooncake 与 InfiniBand 传输 KV，最前端的 Router 统一对外暴露 HTTP 接口。

Client → Router (:8192) → Prefill (:23456) / Decode (:23457)

PD 分离部署的最小单元是 prefill 和 Decode 实例各用1台机器。可根据上下文长度和性能要求，适当增加 Prefill 实例的数量，使 Prefill 和 Decode 的吞吐相匹配。建议如下：

输入长度	拓扑
32k以下	1P1D
32k	3P1D
64k	4P1D
96k及以上	3P1D

Prefill启动

prefill启动脚本 start_prefill.sh

#!/bin/bash
# DeepSeek V4 MUSA PD Disaggregation - Prefill Node (统一启动脚本)
#
# Usage:
#   bash start_prefill.sh [MODEL_PATH]
#
# Example:
#   bash start_prefill.sh
#   bash start_prefill.sh /data/models/DeepSeek-V4-Flash-0731-FP8-mt

set -euo pipefail

# 自动获取本机 IP（优先走默认路由的网卡地址）
get_host_ip() {
    ip -4 route get 1.1.1.1 2>/dev/null | awk '{for (i = 1; i <= NF; i++) if ($i == "src") { print $(i + 1); exit }}'
}
HOST_IP=$(get_host_ip)
if [[ -z "${HOST_IP}" ]]; then
    HOST_IP=$(hostname -I | awk '{print $1}')
fi
if [[ -z "${HOST_IP}" ]]; then
    echo "ERROR: 无法自动获取本机 IP" >&2
    exit 1
fi

MODEL_PATH="${1:-/data/models/DeepSeek-V4-Flash-0731-FP8-mt}"
PREFILL_IP="${HOST_IP}:29500"

DEEP_EP_CONFIG='{"normal_dispatch":{"num_sms":60,"num_max_nvl_chunked_send_tokens":26},"normal_combine":{"num_sms":60,"num_max_nvl_chunked_send_tokens":16}}'

echo "HOST_IP=${HOST_IP}  MODEL_PATH=${MODEL_PATH}"

# IB 设备自动检测（200G / 400G）
IB_DEVS=()
for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        rate=$(cat "$port/rate" 2>/dev/null || true)
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
if ((${#IB_DEVS[@]} == 0)); then
    echo "ERROR: 未检测到 200/400Gb IB 设备" >&2
    exit 1
fi
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
SGLANG_IB_DEVICES=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
export MCCL_IB_HCA="$SGLANG_IB_DEVICES"

export SGLANG_DSV4_FP4_EXPERTS=0
export SGLANG_OPT_USE_FUSED_HASH_TOPK=1
export SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK=1
export SGLANG_JIT_DEEPGEMM_PRECOMPILE=0
export SGLANG_DSV4_MUSA_MOE_EXPERIMENTAL=1
export SGLANG_OPT_USE_JIT_INDEXER_METADATA=1
export SGLANG_OPT_USE_JIT_NORM=0
export SGLANG_OPT_FLASHMLA_SPARSE_PREFILL=0
export SGLANG_DEEP_GEMM_BLOCK_M=256

# MHC
export SGLANG_OPT_USE_TILELANG_MHC_PRE=1
export SGLANG_OPT_MHC_PRE_BACKEND=auto
export SGLANG_OPT_DEEPGEMM_HC_PRENORM=1
export SGLANG_OPT_DEEPGEMM_HC_PRENORM_SPLIT_K=32
export SGLANG_OPT_MHC_PRE_BIG_FUSE_THREADS=0
export SGLANG_OPT_MHC_PRE_BIG_FUSE_HIDDEN_BLOCK=0
export SGLANG_OPT_MHC_PRE_BIG_FUSE_PASS_CONFIG=auto
export SGLANG_OPT_MHC_PRENORM_BACKEND=deepgemm
export SGLANG_OPT_MHC_PRENORM_SPLIT_K=32
export SGLANG_OPT_MHC_PRENORM_TILELANG_IMPL=auto
export SGLANG_OPT_USE_TILELANG_MHC_POST=1
export SGLANG_OPT_MHC_POST_THREADS=0
export SGLANG_OPT_MHC_POST_HIDDEN_BLOCK=0
export SGLANG_OPT_MHC_POST_PASS_CONFIG=auto
export SGLANG_OPT_MHC_POST_LAYOUT=auto
export SGLANG_OPT_MHC_POST_DIRECT_STORE=0

# GEMM / HC head
export SGLANG_OPT_BF16_FP32_GEMM_ALGO=deep_gemm
export SGLANG_OPT_HC_HEAD_TILELANG=1

# TopK
export SGLANG_DEEPSEEK_V4_MUSA_ENABLE_JIT_TOPK512=1

# RoPE / Hadamard
export SGLANG_OPT_DSV4_MUSA_TILELANG_ROPE=0
export SGLANG_OPT_DSV4_MUSA_FUSED_ROPE_HADAMARD=1
export SGLANG_OPT_DSV4_MUSA_FUSED_NSA_ROPE_HADAMARD=1
export SGLANG_OPT_DSV4_MUSA_TILELANG_WO_A=0

# Quant
export SGLANG_OPT_USE_TILEKERNELS_FP8_QUANT=1
export SGLANG_OPT_USE_TILEKERNELS_SWIGLU_QUANT=1

# MTP
export SGLANG_PREP_IN_CUDA_GRAPH=0
export SGLANG_DSV4_EAGLE_DRAFT_EXTEND_CUDA_GRAPH=0
export SGLANG_DSV4_HC_HEAD_TP=0

# KV transfer
export SGLANG_DISAGGREGATION_QUEUE_SIZE=32
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=64

# c128 online
export SGLANG_DSV4_COMPRESS_STATE_DTYPE=bf16
export SGLANG_OPT_USE_ONLINE_COMPRESS=1
export SGLANG_OPT_USE_COMPRESSOR_V2_MUSA=1
export SGLANG_DISAGGREGATION_ALL_CP_RANKS_TRANSFER=1

export SGLANG_HEALTH_CHECK_TIMEOUT=1200
export DEEPEP_CPU_POLL_TIMEOUT=200000

python -m sglang.launch_server \
    --trust-remote-code \
    --model-path "$MODEL_PATH" \
    --tp 8 \
    --ep 8 \
    --mem-fraction-static 0.78 \
    --swa-full-tokens-ratio 0.2 \
    --host 0.0.0.0 \
    --port 23456 \
    --chunked-prefill-size 40960 \
    --max-prefill-tokens 65536 \
    --disable-overlap-schedule \
    --log-level info \
    --watchdog-timeout 6000 \
    --disable-cuda-graph \
    --enable-nsa-prefill-context-parallel \
    --nsa-prefill-cp-mode round-robin-split \
    --moe-a2a-backend deepep \
    --deepep-mode normal \
    --deepep-config "$DEEP_EP_CONFIG" \
    --disaggregation-mode prefill \
    --disaggregation-transfer-backend mooncake \
    --disaggregation-bootstrap-port 8998 \
    --disaggregation-ib-device "${SGLANG_IB_DEVICES}" \
    --reasoning-parser deepseek-v4 \
    --tool-call-parser deepseekv4 \
    --enable-metrics \
    --dist-init-addr "$PREFILL_IP"

---

Decode启动

decode启动脚本 start_decode.sh

#!/bin/bash
# DeepSeek V4 MUSA PD Disaggregation - Decode Node (Attn DP8 + MoE TP8)
#
# Usage:
#   bash start_decode.sh [MODEL_PATH]
#
# Example:
#   bash start_decode.sh
#   bash start_decode.sh /data/models/DeepSeek-V4-Flash-0731-FP8-mt

set -euo pipefail

# 自动获取本机 IP（优先走默认路由的网卡地址）
get_host_ip() {
    ip -4 route get 1.1.1.1 2>/dev/null | awk '{for (i = 1; i <= NF; i++) if ($i == "src") { print $(i + 1); exit }}'
}
HOST_IP=$(get_host_ip)
if [[ -z "${HOST_IP}" ]]; then
    HOST_IP=$(hostname -I | awk '{print $1}')
fi
if [[ -z "${HOST_IP}" ]]; then
    echo "ERROR: 无法自动获取本机 IP" >&2
    exit 1
fi

MODEL_PATH="${1:-/data/models/DeepSeek-V4-Flash-0731-FP8-mt}"
DECODER_IP="${HOST_IP}:29500"

echo "HOST_IP=${HOST_IP}  MODEL_PATH=${MODEL_PATH}"

# IB 设备自动检测（200G / 400G）
IB_DEVS=()
for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        rate=$(cat "$port/rate" 2>/dev/null || true)
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
if ((${#IB_DEVS[@]} == 0)); then
    echo "ERROR: 未检测到 200/400Gb IB 设备" >&2
    exit 1
fi
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
SGLANG_IB_DEVICES=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
export MCCL_IB_HCA="$SGLANG_IB_DEVICES"

export SGLANG_DSV4_FP4_EXPERTS=0
export SGLANG_OPT_USE_FUSED_HASH_TOPK=1
export SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK=1
export SGLANG_JIT_DEEPGEMM_PRECOMPILE=1
export SGLANG_OPT_USE_JIT_EP_ACTIVATION=1
export SGLANG_DEEP_GEMM_BLOCK_M=256

# MHC
export SGLANG_OPT_USE_TILELANG_MHC_PRE=1
export SGLANG_OPT_MHC_PRE_BACKEND=auto
export SGLANG_OPT_DEEPGEMM_HC_PRENORM=1
export SGLANG_OPT_DEEPGEMM_HC_PRENORM_SPLIT_K=32
export SGLANG_OPT_MHC_PRE_BIG_FUSE_THREADS=0
export SGLANG_OPT_MHC_PRE_BIG_FUSE_HIDDEN_BLOCK=0
export SGLANG_OPT_MHC_PRE_BIG_FUSE_PASS_CONFIG=auto
export SGLANG_OPT_MHC_PRENORM_BACKEND=deepgemm
export SGLANG_OPT_MHC_PRENORM_SPLIT_K=32
export SGLANG_OPT_MHC_PRENORM_TILELANG_IMPL=auto
export SGLANG_OPT_USE_TILELANG_MHC_POST=1
export SGLANG_OPT_MHC_POST_THREADS=0
export SGLANG_OPT_MHC_POST_HIDDEN_BLOCK=0
export SGLANG_OPT_MHC_POST_PASS_CONFIG=auto
export SGLANG_OPT_MHC_POST_LAYOUT=auto
export SGLANG_OPT_MHC_POST_DIRECT_STORE=0

# GEMM / HC head
export SGLANG_OPT_BF16_FP32_GEMM_ALGO=deep_gemm
export SGLANG_OPT_HC_HEAD_TILELANG=0

# TopK
export SGLANG_DEEPSEEK_V4_MUSA_ENABLE_JIT_TOPK512=1

# RoPE / Hadamard
export SGLANG_OPT_DSV4_MUSA_TILELANG_ROPE=0
export SGLANG_OPT_DSV4_MUSA_FUSED_ROPE_HADAMARD=1
export SGLANG_OPT_DSV4_MUSA_FUSED_NSA_ROPE_HADAMARD=1
export SGLANG_OPT_DSV4_MUSA_TILELANG_WO_A=0

# Quant
export SGLANG_OPT_USE_TILEKERNELS_FP8_QUANT=1
export SGLANG_OPT_USE_TILEKERNELS_SWIGLU_QUANT=1

# MTP / EAGLE graph preparation optimizations.
export SGLANG_PREP_IN_CUDA_GRAPH=1
export SGLANG_DSV4_EAGLE_DRAFT_EXTEND_CUDA_GRAPH=1
export SGLANG_DSV4_HC_HEAD_TP=0

# KV transfer
export SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS=128

# c128 online
export SGLANG_DSV4_COMPRESS_STATE_DTYPE=bf16
export SGLANG_OPT_USE_ONLINE_COMPRESS=1
export SGLANG_OPT_USE_COMPRESSOR_V2_MUSA=1
export SGLANG_DISAGGREGATION_ALL_CP_RANKS_TRANSFER=1

# Decode Overlap
export SGLANG_FUSE_VERIFY_SAMPLE=1
export SGLANG_OPT_DSV4_MUSA_STRIDED_QKV_RMSNORM=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1

python -m sglang.launch_server \
    --trust-remote-code \
    --model-path "$MODEL_PATH" \
    --tp 8 \
    --max-running-requests 144 \
    --num-reserved-decode-tokens 256 \
    --disaggregation-mode decode \
    --disaggregation-transfer-backend mooncake \
    --disaggregation-bootstrap-port 8998 \
    --disaggregation-ib-device "${SGLANG_IB_DEVICES}" \
    --dist-init-addr "$DECODER_IP" \
    --mem-fraction-static 0.8 \
    --swa-full-tokens-ratio 0.35 \
    --host 0.0.0.0 \
    --port 23457 \
    --cuda-graph-max-bs 128 \
    --chunked-prefill-size 8192 \
    --skip-server-warmup \
    --speculative-algorithm DSPARK \
    --speculative-dspark-block-size 5 \
    --dp 8 \
    --enable-dp-attention \
    --enable-dp-lm-head \
    --reasoning-parser deepseek-v4 \
    --tool-call-parser deepseekv4 \
    --enable-metrics

---

开启 HiCache

• 开启 L2 HiCache，Prefill 节点启动服务时增加如下参数：

--enable-hierarchical-cache \
--hicache-ratio 2 \
--hicache-size 0 \
--hicache-write-policy write_through \
--hicache-io-backend direct \
--hicache-mem-layout page_first_direct \

• 开启 L3 Hicache，需要使用 mooncake store，按如下步骤配置。

1. 在某一个 Prefill 节点 docker 内执行命令，启动 mooncake_master（是否使用 etcd 对应不同的命令）：

# 不使用 etcd 的启动命令
mooncake_master \
  --enable_http_metadata_server=true \
  --http_metadata_server_port=8080 \
  --rpc_port=50051 \
  --eviction_high_watermark_ratio=0.95

# 使用 etcd 的启动命令，其中 etcd_endpoints 为 etcd 地址，rpc_address为 master 服务本机IP
mooncake_master \
  --enable_ha=true \
  --etcd_endpoints="etcd_ip:etcd_port" \
  --rpc_port=50051 \
  --rpc_address=etcd_ip \
  --eviction_high_watermark_ratio=0.95

2. 每个 Prefill 节点启动服务时，在 L2 HiCache 改动基础上，增加环境变量和启动参数。

   下面两套环境变量按步骤 1 的选择二选一，不要同时写入：

   • 方案 A（不使用 etcd，对应步骤 1 的第一条命令）：mooncake_master_ip 为步骤 1 中启动 mooncake master 的节点 IP。

# 以下环境变量在 SGLANG_IB_DEVICES=$(echo "$IB_DEVS_SORTED" | paste -sd, -)之后添加
export MOONCAKE_DEVICE="${SGLANG_IB_DEVICES}"
export MOONCAKE_PROTOCOL="${MOONCAKE_PROTOCOL:-rdma}"
export MOONCAKE_GLOBAL_SEGMENT_SIZE="${MOONCAKE_GLOBAL_SEGMENT_SIZE:-16gb}"
export MOONCAKE_MASTER='mooncake_master_ip:50051'
export MOONCAKE_TE_META_DATA_SERVER="http://mooncake_master_ip:8080/metadata"

• 方案 B（使用 etcd，对应步骤 1 的第二条命令）：etcd_ip:etcd_port 为实际使用的 etcd 地址和端口。

# 以下环境变量在 SGLANG_IB_DEVICES=$(echo "$IB_DEVS_SORTED" | paste -sd, -)之后添加
export MOONCAKE_DEVICE="${SGLANG_IB_DEVICES}"
export MOONCAKE_PROTOCOL="${MOONCAKE_PROTOCOL:-rdma}"
export MOONCAKE_GLOBAL_SEGMENT_SIZE="${MOONCAKE_GLOBAL_SEGMENT_SIZE:-16gb}"
export MOONCAKE_MASTER='etcd://etcd_ip:etcd_port'
export MOONCAKE_TE_META_DATA_SERVER='etcd://etcd_ip:etcd_port'

3. 无论选择方案 A 还是方案 B，都需要在 SGLang 服务启动命令中增加如下参数：

# 以下参数在 SGLang 服务启动命令中添加
--enable-hierarchical-cache \
--hicache-ratio 2 \
--hicache-size 0 \
--hicache-write-policy write_through \
--hicache-io-backend direct \
--hicache-mem-layout page_first_direct \
--hicache-storage-backend mooncake

4. 拉起服务后，会看到如下 Reuse initialized mooncake transfer engine 和 Mooncake store warmup successfully 的日志。

参考启动命令

以下示例命令启动 3个 Prefill 实例，1个 Decode 实例。如果只需要1个 Prefill 实例，则只需启动 prefill node 0，启动 Router 节点也只需第一个 --prefill 参数。

Prefill 节点：

# prefill node 0 执行
bash start_prefill.sh /data/models/DeepSeek-V4-Flash-0731-FP8-mt
# prefill node 1 执行
bash start_prefill.sh /data/models/DeepSeek-V4-Flash-0731-FP8-mt
# prefill node 2 执行
bash start_prefill.sh /data/models/DeepSeek-V4-Flash-0731-FP8-mt

Decode 节点：

# decode node 执行
bash start_decode.sh /data/models/DeepSeek-V4-Flash-0731-FP8-mt

Router 节点：

先把 4 个节点 IP 写成变量（不要在命令里直接写 <prefill_0_ip> 这类尖括号占位符，bash 会把 < 当成输入重定向并直接报语法错误）：

export P0=10.0.0.11  P1=10.0.0.12  P2=10.0.0.13   # 3 个 Prefill 节点
export D0=10.0.0.21                               # Decode 节点

# prefill node 0 上执行
python3 -m sglang_router.launch_router   \
  --pd-disaggregation \
  --prefill http://"$P0":23456 8998 \
  --prefill http://"$P1":23456 8998 \
  --prefill http://"$P2":23456 8998 \
  --decode http://"$D0":23457 \
  --host 0.0.0.0 \
  --port 8192 \
  --prefill-policy round_robin \
  --decode-policy round_robin \
  --worker-startup-timeout-secs 1200 \
  --health-check-timeout-secs 1200 \
  --prometheus-port 29001

---

单机部署

服务启动脚本参考

提供针对 DeepSeek-V4-Flash-0731-FP8 3种并行策略的示例脚本，适用场景分别如下。

• TP8+EP8+CP：长上下文，追求 prefill 低时延，不支持 DSpark。

• TP8+EP8+DP8：追求高并发大吞吐。

• 纯TP：追求单请求的低延迟。

服务启动脚本 start_server.sh，TP8+EP8+CP并行

# IB 设备自动检测（200G / 400G）
IB_DEVS=()
for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        rate=$(cat "$port/rate" 2>/dev/null || true)
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
if ((${#IB_DEVS[@]} == 0)); then
    echo "ERROR: 未检测到 200/400Gb IB 设备" >&2
    exit 1
fi
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
SGLANG_IB_DEVICES=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
export MCCL_IB_HCA="$SGLANG_IB_DEVICES"

MODEL_PATH="${1:-/data/models/DeepSeek-V4-Flash-0731-FP8-mt}"

export SGLANG_DSV4_FP4_EXPERTS=0
export SGLANG_OPT_USE_FUSED_HASH_TOPK=1
export SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK=1
export SGLANG_JIT_DEEPGEMM_PRECOMPILE=0
export SGLANG_DSV4_MUSA_MOE_EXPERIMENTAL=1
export SGLANG_OPT_USE_JIT_INDEXER_METADATA=1
export SGLANG_OPT_USE_JIT_NORM=0
export SGLANG_OPT_FLASHMLA_SPARSE_PREFILL=0
export SGLANG_DEEP_GEMM_BLOCK_M=256

# MHC
export SGLANG_OPT_USE_TILELANG_MHC_PRE=1
export SGLANG_OPT_MHC_PRE_BACKEND=auto
export SGLANG_OPT_DEEPGEMM_HC_PRENORM=1
export SGLANG_OPT_DEEPGEMM_HC_PRENORM_SPLIT_K=32
export SGLANG_OPT_MHC_PRE_BIG_FUSE_THREADS=0
export SGLANG_OPT_MHC_PRE_BIG_FUSE_HIDDEN_BLOCK=0
export SGLANG_OPT_MHC_PRE_BIG_FUSE_PASS_CONFIG=auto
export SGLANG_OPT_MHC_PRENORM_BACKEND=deepgemm
export SGLANG_OPT_MHC_PRENORM_SPLIT_K=32
export SGLANG_OPT_MHC_PRENORM_TILELANG_IMPL=auto
export SGLANG_OPT_USE_TILELANG_MHC_POST=1
export SGLANG_OPT_MHC_POST_THREADS=0
export SGLANG_OPT_MHC_POST_HIDDEN_BLOCK=0
export SGLANG_OPT_MHC_POST_PASS_CONFIG=auto
export SGLANG_OPT_MHC_POST_LAYOUT=auto
export SGLANG_OPT_MHC_POST_DIRECT_STORE=0

# GEMM / HC head
export SGLANG_OPT_BF16_FP32_GEMM_ALGO=deep_gemm
export SGLANG_OPT_HC_HEAD_TILELANG=1

# TopK
export SGLANG_DEEPSEEK_V4_MUSA_ENABLE_JIT_TOPK512=1

# RoPE / Hadamard
export SGLANG_OPT_DSV4_MUSA_TILELANG_ROPE=0
export SGLANG_OPT_DSV4_MUSA_FUSED_ROPE_HADAMARD=1
export SGLANG_OPT_DSV4_MUSA_FUSED_NSA_ROPE_HADAMARD=1
export SGLANG_OPT_DSV4_MUSA_TILELANG_WO_A=0

# Quant
export SGLANG_OPT_USE_TILEKERNELS_FP8_QUANT=1
export SGLANG_OPT_USE_TILEKERNELS_SWIGLU_QUANT=1

# MTP
export SGLANG_PREP_IN_CUDA_GRAPH=0
export SGLANG_DSV4_EAGLE_DRAFT_EXTEND_CUDA_GRAPH=0
export SGLANG_DSV4_HC_HEAD_TP=0

# KV transfer
export SGLANG_DISAGGREGATION_QUEUE_SIZE=32
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=64

# c128 online
export SGLANG_DSV4_COMPRESS_STATE_DTYPE=bf16
export SGLANG_OPT_USE_ONLINE_COMPRESS=1
export SGLANG_OPT_USE_COMPRESSOR_V2_MUSA=1
export SGLANG_DISAGGREGATION_ALL_CP_RANKS_TRANSFER=1

export SGLANG_HEALTH_CHECK_TIMEOUT=1200
export DEEPEP_CPU_POLL_TIMEOUT=200000

python -m sglang.launch_server \
    --trust-remote-code \
    --model-path "$MODEL_PATH" \
    --tp 8 \
    --ep 8 \
    --mem-fraction-static 0.78 \
    --swa-full-tokens-ratio 0.2 \
    --host 0.0.0.0 \
    --port 24586 \
    --chunked-prefill-size 40960 \
    --max-prefill-tokens 65536 \
    --disable-overlap-schedule \
    --log-level info \
    --watchdog-timeout 6000 \
    --disable-cuda-graph \
    --enable-nsa-prefill-context-parallel \
    --nsa-prefill-cp-mode round-robin-split \
    --moe-a2a-backend deepep \
    --deepep-mode auto \
    --reasoning-parser deepseek-v4 \
    --tool-call-parser deepseekv4 \
    --enable-metrics

服务启动脚本 start_server.sh，TP8+EP8+DP8 并行

# IB 设备自动检测（检测 200G/400G IB 设备）
IB_DEVS=()
for dev in /sys/class/infiniband/*; do
    [[ -e "$dev" ]] || continue
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        rate=$(cat "$port/rate" 2>/dev/null || true)
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
SGLANG_IB_DEVICES=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
MCCL_IB_HCA="$SGLANG_IB_DEVICES"
export MCCL_IB_HCA

MODEL_PATH="${1:-/data/models/DeepSeek-V4-Flash-0731-FP8-mt}"

export SGLANG_DSV4_FP4_EXPERTS=0
export SGLANG_OPT_USE_FUSED_HASH_TOPK=1
export SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK=1
export SGLANG_JIT_DEEPGEMM_PRECOMPILE=1
export SGLANG_OPT_USE_JIT_EP_ACTIVATION=1
export SGLANG_DEEP_GEMM_BLOCK_M=256

# MHC
export SGLANG_OPT_USE_TILELANG_MHC_PRE=1
export SGLANG_OPT_MHC_PRE_BACKEND=auto
export SGLANG_OPT_DEEPGEMM_HC_PRENORM=1
export SGLANG_OPT_DEEPGEMM_HC_PRENORM_SPLIT_K=32
export SGLANG_OPT_MHC_PRE_BIG_FUSE_THREADS=0
export SGLANG_OPT_MHC_PRE_BIG_FUSE_HIDDEN_BLOCK=0
export SGLANG_OPT_MHC_PRE_BIG_FUSE_PASS_CONFIG=auto
export SGLANG_OPT_MHC_PRENORM_BACKEND=deepgemm
export SGLANG_OPT_MHC_PRENORM_SPLIT_K=32
export SGLANG_OPT_MHC_PRENORM_TILELANG_IMPL=auto
export SGLANG_OPT_USE_TILELANG_MHC_POST=1
export SGLANG_OPT_MHC_POST_THREADS=0
export SGLANG_OPT_MHC_POST_HIDDEN_BLOCK=0
export SGLANG_OPT_MHC_POST_PASS_CONFIG=auto
export SGLANG_OPT_MHC_POST_LAYOUT=auto
export SGLANG_OPT_MHC_POST_DIRECT_STORE=0

# GEMM / HC head
export SGLANG_OPT_BF16_FP32_GEMM_ALGO=deep_gemm
export SGLANG_OPT_HC_HEAD_TILELANG=0

# TopK
export SGLANG_DEEPSEEK_V4_MUSA_ENABLE_JIT_TOPK512=1

# RoPE / Hadamard
export SGLANG_OPT_DSV4_MUSA_TILELANG_ROPE=0
export SGLANG_OPT_DSV4_MUSA_FUSED_ROPE_HADAMARD=1
export SGLANG_OPT_DSV4_MUSA_FUSED_NSA_ROPE_HADAMARD=1
export SGLANG_OPT_DSV4_MUSA_TILELANG_WO_A=0

# Quant
export SGLANG_OPT_USE_TILEKERNELS_FP8_QUANT=1
export SGLANG_OPT_USE_TILEKERNELS_SWIGLU_QUANT=1

# MTP / EAGLE graph preparation optimizations.
export SGLANG_PREP_IN_CUDA_GRAPH=1
export SGLANG_DSV4_EAGLE_DRAFT_EXTEND_CUDA_GRAPH=1
export SGLANG_DSV4_HC_HEAD_TP=0

# KV transfer
export SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS=128

# c128 online
export SGLANG_DSV4_COMPRESS_STATE_DTYPE=bf16
export SGLANG_OPT_USE_ONLINE_COMPRESS=1
export SGLANG_OPT_USE_COMPRESSOR_V2_MUSA=1
export SGLANG_DISAGGREGATION_ALL_CP_RANKS_TRANSFER=1

# Decode Overlap
export SGLANG_FUSE_VERIFY_SAMPLE=1
export SGLANG_OPT_DSV4_MUSA_STRIDED_QKV_RMSNORM=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1

export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256

export SGLANG_HEALTH_CHECK_TIMEOUT=1200
export DEEPEP_CPU_POLL_TIMEOUT=200000

python -m sglang.launch_server \
    --trust-remote-code \
    --model-path "$MODEL_PATH" \
    --tp 8 \
    --ep 8 \
    --mem-fraction-static 0.8 \
    --host 0.0.0.0 \
    --port 24586 \
    --chunked-prefill-size 40960 \
    --max-prefill-tokens 60000 \
    --disable-overlap-schedule \
    --log-level info \
    --tokenizer-backend fastokens \
    --watchdog-timeout 6000 \
    --moe-a2a-backend deepep \
    --deepep-mode auto \
    --cuda-graph-max-bs 128 \
    --dp 8 \
    --enable-dp-attention \
    --enable-dp-lm-head \
    --speculative-algorithm DSPARK \
    --speculative-dspark-block-size 5

服务启动脚本 start_server.sh，纯 TP 并行

#!/bin/bash
# DeepSeek V4 MUSA PD agg

# IB 设备自动检测（检测 200G/400G IB 设备）
IB_DEVS=()
for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")
    for port in "$dev"/ports/*; do
        rate=$(cat "$port/rate" 2>/dev/null)
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
SGLANG_IB_DEVICES=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
MCCL_IB_HCA="$SGLANG_IB_DEVICES"
export MCCL_IB_HCA

MODEL_PATH=$1

export SGLANG_DSV4_FP4_EXPERTS=0
export SGLANG_OPT_USE_FUSED_HASH_TOPK=1
export SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK=1
export SGLANG_JIT_DEEPGEMM_PRECOMPILE=1
export SGLANG_OPT_USE_JIT_EP_ACTIVATION=1
export SGLANG_DEEP_GEMM_BLOCK_M=256

# MHC
export SGLANG_OPT_USE_TILELANG_MHC_PRE=1
export SGLANG_OPT_MHC_PRE_BACKEND=auto
export SGLANG_OPT_DEEPGEMM_HC_PRENORM=1
export SGLANG_OPT_DEEPGEMM_HC_PRENORM_SPLIT_K=32
export SGLANG_OPT_MHC_PRE_BIG_FUSE_THREADS=0
export SGLANG_OPT_MHC_PRE_BIG_FUSE_HIDDEN_BLOCK=0
export SGLANG_OPT_MHC_PRE_BIG_FUSE_PASS_CONFIG=auto
export SGLANG_OPT_MHC_PRENORM_BACKEND=deepgemm
export SGLANG_OPT_MHC_PRENORM_SPLIT_K=32
export SGLANG_OPT_MHC_PRENORM_TILELANG_IMPL=auto
export SGLANG_OPT_USE_TILELANG_MHC_POST=1
export SGLANG_OPT_MHC_POST_THREADS=0
export SGLANG_OPT_MHC_POST_HIDDEN_BLOCK=0
export SGLANG_OPT_MHC_POST_PASS_CONFIG=auto
export SGLANG_OPT_MHC_POST_LAYOUT=auto
export SGLANG_OPT_MHC_POST_DIRECT_STORE=0

# GEMM / HC head
export SGLANG_OPT_BF16_FP32_GEMM_ALGO=deep_gemm
export SGLANG_OPT_HC_HEAD_TILELANG=0

# TopK
export SGLANG_DEEPSEEK_V4_MUSA_ENABLE_JIT_TOPK512=1

# RoPE / Hadamard
export SGLANG_OPT_DSV4_MUSA_TILELANG_ROPE=0
export SGLANG_OPT_DSV4_MUSA_FUSED_ROPE_HADAMARD=1
export SGLANG_OPT_DSV4_MUSA_FUSED_NSA_ROPE_HADAMARD=1
export SGLANG_OPT_DSV4_MUSA_TILELANG_WO_A=0

# Quant
export SGLANG_OPT_USE_TILEKERNELS_FP8_QUANT=1
export SGLANG_OPT_USE_TILEKERNELS_SWIGLU_QUANT=1

# MTP / EAGLE graph preparation optimizations.
export SGLANG_PREP_IN_CUDA_GRAPH=1
export SGLANG_DSV4_EAGLE_DRAFT_EXTEND_CUDA_GRAPH=1
export SGLANG_DSV4_HC_HEAD_TP=0

# KV transfer
export SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS=128

# c128 online
export SGLANG_DSV4_COMPRESS_STATE_DTYPE=bf16
export SGLANG_OPT_USE_ONLINE_COMPRESS=1
export SGLANG_OPT_USE_COMPRESSOR_V2_MUSA=1
export SGLANG_DISAGGREGATION_ALL_CP_RANKS_TRANSFER=1

# Decode Overlap
export SGLANG_FUSE_VERIFY_SAMPLE=1
export SGLANG_OPT_DSV4_MUSA_STRIDED_QKV_RMSNORM=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1

export SGLANG_HEALTH_CHECK_TIMEOUT=1200

python -m sglang.launch_server \
    --trust-remote-code \
    --model-path "$MODEL_PATH" \
    --tp 8 \
    --mem-fraction-static 0.8 \
    --host 0.0.0.0 \
    --port 24586 \
    --chunked-prefill-size 8192 \
    --disable-overlap-schedule \
    --log-level info \
    --tokenizer-backend fastokens \
    --watchdog-timeout 6000 \
    --cuda-graph-max-bs 128 \
    --speculative-algorithm DSPARK \
    --speculative-dspark-block-size 5

启动命令参考

bash start_server.sh /data/models/DeepSeek-V4-Flash-0731-FP8-mt