Kimi 模型部署

本章节以 Kimi-K2.6-FP8 部署为例。

• 模型权重下载。

• 参考 环境准备 进行环境检查、启动容器。

• 容器镜像地址参考 版本发布信息-镜像地址。

• PD分离脚本涉及的参数说明和一键运行说明见 快速开始。

模型部署

示例采取的是部署方案 P3D6，即3台机器作为Prefill，6台机器为Decode。

Decode 侧启用了 EAGLE3 投机采样，需要额外准备 draft 模型 Kimi-K2.6-Eagle3，下载地址为 Kimi-K2.6-Eagle3。如暂不使用投机采样，可删除 decode_server.sh 中的 --speculative-* 参数与 --speculative-draft-model-path。

• 如果需要修改部署方式，请更新 run.sh 中 PREFILL_SERVER_COUNT 和 DECODE_SERVER_COUNT。

• DeepEP 的配置文件内容参考 DeepEP 配置文件

Prefill启动

prefill启动脚本 prefill_server.sh

#!/bin/bash
export PATH=/root/.virtualenvs/sglang-default/bin/:$PATH

export TP_SOCKET_IFNAME=bond0
export GLOO_SOCKET_IFNAME=bond0

export MCCL_SOCKET_IFNAME=bond0
export MCCL_SOCKET_FAMILY=AF_INET
export MCCL_DEBUG=WARN
# export MCCL_DEBUG_SUBSYS=INIT,BOOTSTRAP,NET,SHM,ENV
export MCCL_TOPO_ENHANCE=0

export MUSA_LAUNCH_BLOCKING=0
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
# export SGLANG_PP_LAYER_PARTITION="20,20,21"
export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:/usr/local/musa/lib:$LD_LIBRARY_PATH
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export SGLANG_USE_DEEPGEMM_BMM=0
export SGLANG_MOE_PADDING=1
export SGLANG_USE_MUSA_FUSED_KERNEL=1
export VLLM_CONFIGURE_LOGGING=0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
export SGLANG_DEEP_GEMM_BLOCK_M=256
export MC_TE_METRIC=true
export MC_GID_INDEX=3
export MC_MTU=4096

# DeepEP
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=200
export MUSA_DEVICE_PAGE_SIZE=0x1000

# ACE
export SGLANG_DEEPEP_USE_MUSA_ACE=1
# shared experts overlap
export SGLANG_SBO_COMBINE_SHARED=1

export SGLANG_MUSA_ENABLE_ATTN_TP_PYNCCL=1 
export MCCL_CHECK_POINTERS=0

NODE_RANK=$1
PREFILL_IP="${2}:12345"
LOG_DIR=$3
MODEL_PATH=$4
HOST_IP=$5
NNODES=$6
echo "NNODES=$NNODES"
IB_DEVS=()

for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")

    for port in "$dev"/ports/*; do
        # 速率（如 200 Gb/sec）
        rate=$(cat "$port/rate" 2>/dev/null)
        # link_layer: InfiniBand / Ethernet (RoCE)
        link=$(cat "$port/link_layer" 2>/dev/null)
        # 只要是 200G/400G（IB 或 RoCE 都收）
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
# 排序 + 去重
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
# # 生成 MCCL / SGLang 变量
# MCCL_IB_HCA=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
# SGLANG_IB_DEVICES="$MCCL_IB_HCA"
# echo "IB_DEVICES=$MCCL_IB_HCA"


# Mooncake/PD 传输使用脚本扫描到的 400G 网卡
SGLANG_IB_DEVICES=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
export SGLANG_IB_DEVICES

# MCCL 暂时只使用所有节点上 GID 3 均有效的 mlx5_1
export MCCL_IB_HCA="=mlx5_1:1"

echo "SGLANG_IB_DEVICES=$SGLANG_IB_DEVICES"
echo "MCCL_IB_HCA=$MCCL_IB_HCA"
echo "MCCL_IB_GID_INDEX=$MCCL_IB_GID_INDEX"


mkdir -p "${LOG_DIR}"

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# DeepEP 配置文件路径：优先使用脚本同目录下的 deepep.config，否则回落到共享配置目录
if [[ -f "${SCRIPT_DIR}/deepep.config" ]]; then
    DEEP_EP_CONFIG="${DEEP_EP_CONFIG:-${SCRIPT_DIR}/deepep.config}"
else
    DEEP_EP_CONFIG="${DEEP_EP_CONFIG:-/data/workspace/shared_configs/deepep/deepep.config}"
fi

PROFILER_DIR_BASE="${LOG_DIR}/traces"
export SGLANG_TORCH_PROFILER_DIR="${PROFILER_DIR_BASE}/sglang_prefill_profiling_P${NODE_RANK}"
export TRANSFORMERS_VERBOSITY=error
# export SGLANG_EXPERT_DISTRIBUTION_RECORDER_DIR="/data/workspace/expert_distribution"

source /root/.virtualenvs/sglang-default/bin/activate
nohup python3 -m sglang.launch_server \
            --model-path $MODEL_PATH \
            --served-model-name KimiK2.6 \
            --trust-remote-code \
            --disable-cuda-graph \
            --disable-overlap-schedule \
            --host 0.0.0.0 \
            --port 24586 \
            --nnodes $NNODES  \
            --node-rank $NODE_RANK \
            --tp-size 8 \
            --pp-size 3 \
            --dp-size 8  \
            --ep-size 8 \
            --moe-a2a-backend deepep \
            --deepep-mode normal \
            --moe-dense-tp-size 1 \
            --enable-dp-attention \
            --enable-dp-lm-head \
            --attention-backend fa3 \
            --chunked-prefill-size 65536 \
            --prefill-attention-backend fa3 \
            --sampling-backend flashinfer \
            --disaggregation-mode prefill \
            --dist-init-addr $PREFILL_IP \
            --enable-cache-report \
            --max-prefill-tokens 81920 \
            --context-length 240000 \
            --load-balance-method auto \
            --deepep-config "$DEEP_EP_CONFIG" \
            --max-running-requests 128 \
            --mem-fraction-static 0.75 \
            --schedule-conservativeness 1 \
            --disable-shared-experts-fusion \
            --enable-single-batch-overlap \
            --tool-call-parser kimi_k2 \
            --reasoning-parser kimi_k2 \
            --pp-max-micro-batch-size 1 \
            --kv-cache-dtype bf16 \
            --mamba-ssm-dtype bfloat16 \
            --watchdog-timeout 1200 \
            --disaggregation-ib-device $SGLANG_IB_DEVICES > "${LOG_DIR}/P${NODE_RANK}_${HOST_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

Decode启动

decoder启动脚本 decode_server.sh

#!/bin/bash
export PATH=/root/.virtualenvs/sglang-default/bin/:$PATH

export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0

export MCCL_SOCKET_IFNAME=bond0
export MCCL_SOCKET_FAMILY=AF_INET
export MCCL_DEBUG=WARN
# export MCCL_DEBUG_SUBSYS=INIT,BOOTSTRAP,NET,SHM,ENV
export MCCL_TOPO_ENHANCE=0

export MUSA_LAUNCH_BLOCKING=0
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export MC_TE_METRIC=true
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=6000
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_MAPPING_IB_DEVICE_TO_GPU=1
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=false
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export SGLANG_MOE_PADDING=1
export SGLANG_USE_DEEPGEMM_BMM=0
export SGLANG_MOE_PADDING=1
export SGLANG_USE_MUSA_FUSED_KERNEL=1
export SGLANG_DEEP_GEMM_BLOCK_M=256
export VLLM_CONFIGURE_LOGGING=0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
export SGLANG_DEEPEP_LOW_LATENCY_USE_NVLINK=1
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:/usr/local/musa/lib:$LD_LIBRARY_PATH
export SGLANG_USE_MTT=0
export MCCL_CHECK_POINTERS=0

# DeepEP
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=200
export NVSHMEM_IB_SL=7
export MUSA_DEVICE_PAGE_SIZE=0x1000
export MUSA_MANAGED_FORCE_DEVICE_ALLOC=1
export NVSHMEM_IBGDA_FORCE_NIC_BUF_MEMTYPE=gpumem
export NVSHMEM_IBGDA_NIC_HANDLER=cpu
export NVSHMEM_IBGDA_NUM_PROXY_PER_PE=2

# SBO
export SGLANG_SBO_COMBINE_SHARED=1 # combine shared 并行

export SGLANG_MUSA_ENABLE_ATTN_TP_PYNCCL=1 

NODE_RANK=$1
DECODER_IP="${2}:4343"
LOG_DIR=$3
MODEL_PATH=$4
HOST_IP=$5
NNODES=$6
echo "NNODES=$NNODES"
IB_DEVS=()

for dev in /sys/class/infiniband/*; do
    ibdev=$(basename "$dev")

    for port in "$dev"/ports/*; do
        # 速率（如 200 Gb/sec）
        rate=$(cat "$port/rate" 2>/dev/null)
        # link_layer: InfiniBand / Ethernet (RoCE)
        link=$(cat "$port/link_layer" 2>/dev/null)
        # 只要是 200G/400G（IB 或 RoCE 都收）
        if [[ "$rate" == *"200 Gb"* || "$rate" == *"400 Gb"* ]]; then
            IB_DEVS+=("$ibdev")
            break
        fi
    done
done
# 排序 + 去重
IB_DEVS_SORTED=$(printf "%s\n" "${IB_DEVS[@]}" | sort -V | uniq)
# # 生成 MCCL / SGLang 变量
# MCCL_IB_HCA=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
# SGLANG_IB_DEVICES="$MCCL_IB_HCA"
# echo "IB_DEVICES=$SGLANG_IB_DEVICES"

# Mooncake/PD 传输使用脚本扫描到的 400G 网卡
SGLANG_IB_DEVICES=$(echo "$IB_DEVS_SORTED" | paste -sd, -)
export SGLANG_IB_DEVICES

# MCCL 暂时只使用所有节点上 GID 3 均有效的 mlx5_1
export MCCL_IB_HCA="=mlx5_1:1"

echo "SGLANG_IB_DEVICES=$SGLANG_IB_DEVICES"
echo "MCCL_IB_HCA=$MCCL_IB_HCA"
echo "MCCL_IB_GID_INDEX=$MCCL_IB_GID_INDEX"


SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

PROFILER_DIR_BASE="${LOG_DIR}/traces"
export SGLANG_TORCH_PROFILER_DIR="${PROFILER_DIR_BASE}/sglang_decode_profiling_D${NODE_RANK}"
export TRANSFORMERS_VERBOSITY=error

source /root/.virtualenvs/sglang-default/bin/activate
nohup python3 -m sglang.launch_server \
        --model $MODEL_PATH \
        --trust-remote-code \
        --host 0.0.0.0 \
        --port 23457 \
        --tp-size 48 \
        --ep-size 48 \
        --dp-size 48 \
        --pp-size 1 \
        --mem-fraction-static 0.90 \
        --moe-a2a-backend deepep \
        --deepep-mode low_latency \
        --moe-dense-tp-size 1 \
        --disable-shared-experts-fusion \
        --enable-dp-lm-head \
        --enable-dp-attention \
        --schedule-conservativeness 0.8 \
        --attention-backend fa3 \
        --sampling-backend flashinfer \
        --dist-init-addr $DECODER_IP \
        --nnodes $NNODES \
        --node-rank $NODE_RANK \
        --chunked-prefill-size -1 \
        --prefill-round-robin-balance \
        --enable-single-batch-overlap \
        --load-balance-method auto \
        --disaggregation-mode decode \
        --disaggregation-ib-device $SGLANG_IB_DEVICES \
        --cuda-graph-bs 1 2 3 4 5 \
        --max-total-tokens 140000 \
        --context-length 240000 \
        --max-running-requests 1280 \
        --kv-cache-dtype bf16 \
        --tool-call-parser kimi_k2 \
        --reasoning-parser kimi_k2 \
        --speculative-algorithm EAGLE3  \
        --speculative-num-steps 1 \
        --speculative-eagle-topk 1 \
        --speculative-num-draft-tokens 2 \
        --mamba-ssm-dtype bfloat16 \
        --speculative-draft-model-path /data/models/Kimi-K2.6-Eagle3 \
        --speculative-draft-load-format auto > "${LOG_DIR}/D${NODE_RANK}_${HOST_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

Router启动

Router 启动脚本 router.sh

#!/bin/bash
# Router 启动脚本
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"

PREFILL_IP="http://$1:24586"
DECODE_IP="http://$2:23457"
LOG_DIR=$3

ulimit -n 65535
nohup "${SGLANG_PYTHON}" -m sglang_router.launch_router \
    --pd-disaggregation \
    --mini-lb \
    --prefill $PREFILL_IP \
    --decode $DECODE_IP \
    --host 0.0.0.0 \
    --request-timeout-secs 7200 \
    --port 31000 > "${LOG_DIR}/router_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

创建 hostfile 文件

hostfile文件示例

192.168.100.101
192.168.100.102
192.168.100.103
192.168.100.104
192.168.100.105
192.168.100.106
192.168.100.107
192.168.100.108
192.168.100.109

一键启动所有服务

一键运行脚本 run.sh

#!/bin/bash

CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
#echo $CURRENT_TIME
#mkdir -p ./output/$CURRENT_TIME

PREFILL_SERVER_COUNT=${PREFILL_SERVER_COUNT:-3}
DECODER_SERVER_COUNT=${DECODER_SERVER_COUNT:-6}
WORLD_SIZE=$(( (PREFILL_SERVER_COUNT + DECODER_SERVER_COUNT) * 8 ))

echo -e "\033[32mPREFILL_SERVER_COUNT: $PREFILL_SERVER_COUNT, DECODER_SERVER_COUNT: $DECODER_SERVER_COUNT, \033[0m"

set -u
  WORK_HOME="$PWD"
  EXPNAME="P${PREFILL_SERVER_COUNT}_D${DECODER_SERVER_COUNT}_gpus${WORLD_SIZE}"
  SGLANG_CODE_HOME="${SGLANG_CODE_HOME:-/sgl-workspace/sglang}"
  SGLANG_CODE_HOME_ENV="$SGLANG_CODE_HOME"
  MODEL_PATH=${MODEL_PATH:-"/data/models/Kimi-K2.6-FP8/"}
  HOSTFILE=./hostfile
  LOG_DIR=$WORK_HOME/output/$EXPNAME/$CURRENT_TIME/
  PREFILL_SCRIPT_FILE="$WORK_HOME/prefill_server.sh"
  DECODER_SCRIPT_FILE="$WORK_HOME/decode_server.sh"
  ROUTER_SCRIPT_FILE="$WORK_HOME/router.sh"
  SSH_PORT=${SSH_PORT:-62216}
set +u

echo -e "\033[32mWORK_HOME: $WORK_HOME\033[0m"
echo -e "\033[32mMODEL_PATH: $MODEL_PATH\033[0m"
echo -e "\033[32mPREFILL_CRIPT_FILE: $PREFILL_SCRIPT_FILE\033[0m"
echo -e "\033[32mDECODER_SCRIPT_FILE: $DECODER_SCRIPT_FILE\033[0m"

# bash generate_hostfile.sh 
COUNT=0
hostlist=$(grep -v '^#\|^$' $HOSTFILE | awk '{print $1}' | xargs)

# Check if hostlist is empty
if [ -z "$hostlist" ]; then
  echo "Error: hostlist is empty. Please add IP addresses to the hostfile."
  exit 1
fi

PREFILL_MASTER_IP=$(head -n 1 $HOSTFILE)
DECODER_MASTER_IP=$(tail -n $DECODER_SERVER_COUNT $HOSTFILE | head -n1)
echo -e "\033[32mPREFILL_MASTER_IP: $PREFILL_MASTER_IP\033[0m"
echo -e "\033[32mDECODER_MASTER_IP: $DECODER_MASTER_IP\033[0m"


mkdir -p $LOG_DIR
for host in ${hostlist[@]}; do
  echo $host
  if [[ $COUNT -lt $PREFILL_SERVER_COUNT ]]; then
    PREFILL_INDEX=$COUNT
    cmd="bash -c 'cd $WORK_HOME; bash $PREFILL_SCRIPT_FILE $PREFILL_INDEX $PREFILL_MASTER_IP $LOG_DIR $MODEL_PATH $host $PREFILL_SERVER_COUNT'"
    #cmd="bash -c 'cd $WORK_HOME; echo $PATH '"
    echo $cmd
    ssh -p $SSH_PORT -n $host $cmd
  else
    DECODER_INDEX=$((COUNT - PREFILL_SERVER_COUNT))
    cmd="bash -c 'cd $WORK_HOME; bash $DECODER_SCRIPT_FILE $DECODER_INDEX $DECODER_MASTER_IP $LOG_DIR $MODEL_PATH $host $DECODER_SERVER_COUNT'"
    echo $cmd
    ssh -p $SSH_PORT -n $host $cmd
  fi

  ((COUNT++))
done
cmd="bash -c 'cd $WORK_HOME; bash $ROUTER_SCRIPT_FILE $PREFILL_MASTER_IP $DECODER_MASTER_IP $LOG_DIR'"
echo $cmd
ssh -p $SSH_PORT -n $PREFILL_MASTER_IP $cmd
echo -e "Main log file: \033[34m$LOG_DIR\033[0m"

---