跳到主要内容

Qwen3.5 模型部署

本章节包含 Qwen3.5 系列模型的部署脚本，脚本涉及的参数说明、PD 分离一键部署说明见 快速开始。

  • 模型权重下载。

  • 参考 环境准备 进行环境检查、启动容器。

  • 容器镜像地址参考 版本发布信息-镜像地址。

  • PD分离脚本涉及的参数说明和一键运行说明见 快速开始。

  • 参考 环境准备-RDMA 网卡检测 中的脚本，请其保存为 resolve_sglang_pd_ib_devices.sh，与 SGLang 服务启动脚本放在同一路径下。

  • 部分脚本用到的 EPLB_FILE 是利用测试数据场景进行录制，可联系支持工程师获取。实际生产部署请自行录制，也可删除 EPLB_FILE 环境变量及 SGLang 启动参数中的 init-expert-location。

Qwen3.5-0.8B/2B/4B/9B​

本示例以 1卡（TP1）部署 Qwen3.5-9B。脚本同样适用于 Qwen3.5-0.8B/2B/4B 模型，注意修改模型名和权重路径。

启动脚本 qwen35_9b_server.sh

#!/usr/bin/env bash
set -euo pipefail

MODEL_NAME="${MODEL_NAME:-Qwen3.5-9B}"
MODEL_PATH="${MODEL_PATH:-/data/models/Qwen3.5-9B/}"
TP_SIZE="${TP_SIZE:-1}"
PP_SIZE="${PP_SIZE:-1}"
EP_SIZE="${EP_SIZE:-}"
MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.55}"
CUDA_GRAPH_BS_MAX="${CUDA_GRAPH_BS_MAX:-128}"
MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-}"
DISABLE_OVERLAP_SCHEDULE="${DISABLE_OVERLAP_SCHEDULE:-1}"
DISABLE_PIECEWISE_CUDA_GRAPH="${DISABLE_PIECEWISE_CUDA_GRAPH:-0}"

SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-${MODEL_NAME}}"

SGLANG_VENV="${SGLANG_VENV:-/root/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${PYTHON:-${SGLANG_VENV}/bin/python3}}"
if [[ -f "${SGLANG_VENV}/bin/activate" ]]; then
  # shellcheck disable=SC1091
  source "${SGLANG_VENV}/bin/activate"
fi

HOST="${HOST:-${SGLANG_HOST:-0.0.0.0}}"
PORT="${PORT:-${SGLANG_PORT:-32289}}"
DIST_PORT="${DIST_PORT:-2452}"
CHUNKED_PREFILL_SIZE="${CHUNKED_PREFILL_SIZE:--1}"
ENABLE_CACHE_REPORT="${ENABLE_CACHE_REPORT:-0}"
TOKENIZER_BACKEND="${TOKENIZER_BACKEND:-fastokens}"
HEALTH_WAIT_SECONDS="${HEALTH_WAIT_SECONDS:-900}"

export PATH="${SGLANG_VENV}/bin:${PATH}"
export LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu:/usr/local/musa/lib:/usr/local/openmpi/lib:/usr/local/musa/mudnn/lib:/usr/local/mtshmem/lib:${LD_LIBRARY_PATH:-}"
export MUSA_VISIBLE_DEVICES="${MUSA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
export MTHREADS_VISIBLE_DEVICES="${MTHREADS_VISIBLE_DEVICES:-${MUSA_VISIBLE_DEVICES}}"
export MUSA_LAUNCH_BLOCKING="${MUSA_LAUNCH_BLOCKING:-0}"
export MUSA_ENABLE_LLC_OPT="${MUSA_ENABLE_LLC_OPT:-1}"
export MCCL_IB_GID_INDEX="${MCCL_IB_GID_INDEX:-3}"
export MCCL_NET_SHARED_BUFFERS="${MCCL_NET_SHARED_BUFFERS:-0}"
export MCCL_PROTOS="${MCCL_PROTOS:-2}"
export GLOO_SOCKET_IFNAME="${GLOO_SOCKET_IFNAME:-bond0}"
export TP_SOCKET_IFNAME="${TP_SOCKET_IFNAME:-bond0}"
export VLLM_PATCH_MUSA_CUSTOM_OPS="${VLLM_PATCH_MUSA_CUSTOM_OPS:-1}"
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN="${SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN:-1}"
export SGLANG_SET_CPU_AFFINITY="${SGLANG_SET_CPU_AFFINITY:-1}"
export SGLANG_USE_DEEPGEMM_BMM="${SGLANG_USE_DEEPGEMM_BMM:-0}"
export SGLANG_MOE_PADDING="${SGLANG_MOE_PADDING:-1}"
export SGLANG_USE_MUSA_FUSED_KERNEL="${SGLANG_USE_MUSA_FUSED_KERNEL:-1}"

[[ -d "${MODEL_PATH}" ]] || {
  echo "MODEL_PATH does not exist: ${MODEL_PATH}" >&2
  exit 2
}
[[ -x "${SGLANG_PYTHON}" ]] || {
  echo "SGLANG_PYTHON not executable: ${SGLANG_PYTHON}" >&2
  exit 2
}

WORK_HOME="${WORK_HOME:-$PWD}"
CURRENT_TIME="$(date '+%Y%m%d_%H%M%S')"
LOG_DIR="${LOG_DIR:-${WORK_HOME}/logs/service/${MODEL_NAME}/accuracy/${CURRENT_TIME}}"
LOG_FILE="${LOG_FILE:-${LOG_DIR}/sglang_$(date '+%Y%m%d_%H%M%S').log}"
PID_FILE="${PID_FILE:-${LOG_FILE}.pid}"
CMD_FILE="${CMD_FILE:-${LOG_FILE}.cmd}"
ENV_FILE="${ENV_FILE:-${LOG_FILE}.env}"
mkdir -p "${LOG_DIR}" "${WORK_HOME}/traces"

MASTER_IP="$(hostname -I | awk '{print $1}')"
MASTER_IP="${MASTER_IP:-127.0.0.1}"

if command -v lsof >/dev/null 2>&1; then
  mapfile -t old_pids < <(lsof -ti:"${PORT}" || true)
  if ((${#old_pids[@]} > 0)); then
    echo "killing existing processes on port ${PORT}: ${old_pids[*]}"
    kill -9 "${old_pids[@]}" || true
  fi
fi
pkill -9 -f "[s]glang.launch_server.*--port[[:space:]]+${PORT}" || true
pkill -9 -f "[s]glang.launch_server.*${MODEL_PATH}" || true
sleep 3

cd /sgl-workspace/sglang

cmd=(
  "${SGLANG_PYTHON}" -m sglang.launch_server
  --model-path "${MODEL_PATH}"
  --served-model-name "${SERVED_MODEL_NAME}"
  --trust-remote-code
  --tp-size "${TP_SIZE}"
  --pp-size "${PP_SIZE}"
  --host "${HOST}"
  --port "${PORT}"
  --dist-init-addr "${MASTER_IP}:${DIST_PORT}"
  --nnodes 1
  --node-rank 0
  --attention-backend fa3
  --mm-attention-backend fa3
  --linear-attn-backend flashinfer
  --sampling-backend flashinfer
  --moe-runner-backend auto
  --tokenizer-backend fastokens
  --mem-fraction-static "${MEM_FRACTION_STATIC}"
  --cuda-graph-bs $(seq 1 "${CUDA_GRAPH_BS_MAX}")
  --mamba-scheduler-strategy extra_buffer
  --chunked-prefill-size "${CHUNKED_PREFILL_SIZE}"
  --decode-log-interval 1
  --skip-server-warmup
)

if [[ -n "${EP_SIZE}" ]]; then
  cmd+=(--ep-size "${EP_SIZE}")
fi
if [[ -n "${MOE_RUNNER_BACKEND}" ]]; then
  cmd+=(--moe-runner-backend "${MOE_RUNNER_BACKEND}")
fi
if [[ -n "${TOKENIZER_BACKEND}" ]]; then
  cmd+=(--tokenizer-backend "${TOKENIZER_BACKEND}")
fi
if [[ "${DISABLE_OVERLAP_SCHEDULE}" == "1" || "${DISABLE_OVERLAP_SCHEDULE}" == "true" ]]; then
  cmd+=(--disable-overlap-schedule)
fi
cmd+=(--disable-piecewise-cuda-graph)
if [[ "${ENABLE_CACHE_REPORT}" == "1" || "${ENABLE_CACHE_REPORT}" == "true" ]]; then
  cmd+=(--enable-cache-report)
fi
if declare -p EXTRA_ARGS >/dev/null 2>&1; then
  cmd+=("${EXTRA_ARGS[@]}")
fi
cmd+=(
  --reasoning-parser qwen3
  --tool-call-parser qwen3_coder
)

{
  echo "timestamp=$(date '+%F %T %z')"
  echo "model_name=${MODEL_NAME}"
  echo "served_model_name=${SERVED_MODEL_NAME}"
  echo "model_path=${MODEL_PATH}"
  echo "host=${HOST}"
  echo "port=${PORT}"
  echo "tp_size=${TP_SIZE}"
  echo "pp_size=${PP_SIZE}"
  echo "ep_size=${EP_SIZE}"
  echo "cuda_graph_bs=1..${CUDA_GRAPH_BS_MAX}"
  echo "python=${SGLANG_PYTHON}"
  env | sort | grep -E '^(LD_LIBRARY_PATH|MUSA_|MTHREADS_|MCCL_|GLOO_|TP_SOCKET|VLLM_|SGLANG_)=' || true
} >"${ENV_FILE}"

printf '%q ' "${cmd[@]}" >"${CMD_FILE}"
printf '\n' >>"${CMD_FILE}"

echo "starting ${MODEL_NAME} for GSM8K concurrency 128"
echo "model: ${MODEL_PATH}"
echo "port: ${PORT}"
echo "log: ${LOG_FILE}"
echo "cmd: ${CMD_FILE}"

if [[ "${FOREGROUND:-0}" == "1" ]]; then
  exec "${cmd[@]}"
fi

nohup "${cmd[@]}" >"${LOG_FILE}" 2>&1 </dev/null &
server_pid=$!
echo "${server_pid}" >"${PID_FILE}"
disown "${server_pid}" 2>/dev/null || true
echo "pid=${server_pid}"

for _ in $(seq 1 "${HEALTH_WAIT_SECONDS}"); do
  if ! kill -0 "${server_pid}" 2>/dev/null; then
    echo "server process exited before health check passed; tailing log" >&2
    tail -200 "${LOG_FILE}" >&2 || true
    exit 1
  fi
  curl -sf --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null && {
    echo "health OK on port ${PORT}"
    exit 0
  }
  sleep 1
done

echo "server did not become healthy within ${HEALTH_WAIT_SECONDS}s; tailing log" >&2
tail -200 "${LOG_FILE}" >&2 || true
exit 1

Qwen3.5-27B​

本示例使用 2卡（TP2） 部署， 也设置 --tp-size 1 进行单卡部署。

启动脚本 qwen35_27b_server.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
SGLANG_ENABLE_SPECULATIVE="${SGLANG_ENABLE_SPECULATIVE:-1}"

# 清理缓存
rm -rf ~/.triton/cache
rm -rf /tmp/*
# pkill -f "sgl.*"
# mthreads-gmi | grep -E '^[0-9]+\s+[0-9]+\s+' | awk '{print $2}' |
#sort -u | xargs kill -9
# sleep 3

get_master_ip() {
    local hostname_ip=$(hostname -I 2>/dev/null | awk '{print $1}')

    if [ -n "$hostname_ip" ] && [ "$hostname_ip" != "127.0.0.1" ]; then
        echo "$hostname_ip"
        return
    fi

}

CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
SGLANG_PORT="${SGLANG_PORT:-31000}"
#MASTER_IP=127.0.0.1
MASTER_IP=$(get_master_ip)
NODE_RANK=0
WORLD_SIZE=1
WORK_HOME="$PWD"
LOG_DIR=$WORK_HOME/output/$CURRENT_TIME/
mkdir -p "$LOG_DIR"

export SGLANG_TORCH_PROFILER_DIR=$WORK_HOME/traces
MODEL_PATH=/data/models/Qwen3.5-27B-FP8/
cmd=(
      "$SGLANG_PYTHON" -m sglang.launch_server
      --model-path ${MODEL_PATH}
      --trust-remote-code
      --cuda-graph-bs $(seq 1 32)
      --disable-piecewise-cuda-graph
      --disable-overlap-schedule
      --tp-size 2
      --pp-size 1
      --mem-fraction-static 0.78
      --attention-backend fa3
      --linear-attn-backend flashinfer
      --tokenizer-backend fastokens
      --enable-cache-report
      --mm-attention-backend fa3
      --moe-runner-backend auto
      --sampling-backend flashinfer
      --dist-init-addr ${MASTER_IP}:50022
      --nnodes ${WORLD_SIZE}
      --node-rank ${NODE_RANK}
      --chunked-prefill-size 4096
      --mamba-scheduler-strategy extra_buffer
      --max-running-requests 32
      --max-prefill-tokens 8192
      --context-length 16384
      --port ${SGLANG_PORT}
      --host "${SGLANG_HOST}"
      --decode-log-interval 1 
      --reasoning-parser qwen3
      --tool-call-parser qwen3_coder
)

if [[ "$SGLANG_ENABLE_SPECULATIVE" == "1" ]]; then
      cmd+=(
      --speculative-algo NEXTN
      --speculative-num-steps 1
      --speculative-eagle-topk 1
      --speculative-num-draft-tokens 2
      )
fi

nohup "${cmd[@]}" > "${LOG_DIR}/${MASTER_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

echo -e "Main log file: \033[34m$LOG_DIR\033[0m"

Qwen3.5-35B-A3B​

本示例使用 4卡（TP4） 部署。

启动脚本 qwen35_35b_a3b_server.sh

#!/usr/bin/env bash
set -euo pipefail
source /root/.virtualenvs/sglang-default/bin/activate
export PATH=/root/.virtualenvs/sglang-default/bin:${PATH}
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:/usr/local/musa/lib:/usr/local/musa/lib64:/usr/local/openmpi/lib:/usr/local/musa/mudnn/lib:/usr/local/mtshmem/lib:${LD_LIBRARY_PATH:-}
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_USE_DEEPGEMM_BMM=0
export SGLANG_MOE_PADDING=1
export SGLANG_USE_MUSA_FUSED_KERNEL=1
export TOKENIZERS_PARALLELISM=false
export OPENAI_API_KEY=EMPTY

MODEL_PATH=${MODEL_PATH:-/data/models/Qwen3.5-35B-A3B}
PORT=${PORT:-32001}
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="${LOG_DIR:-$WORK_HOME/output/$CURRENT_TIME}"
LOG_FILE="${LOG_FILE:-$LOG_DIR/sglang_${CURRENT_TIME}.log}"
mkdir -p "$LOG_DIR"

nohup /root/.virtualenvs/sglang-default/bin/python3 -m sglang.launch_server \
  --model-path "${MODEL_PATH}" \
  --served-model-name Qwen3.6-35B-A3B \
  --trust-remote-code \
  --tp-size 4   \
  --ep-size 1   \
  --host 0.0.0.0   \
  --port "${PORT}"   \
  --attention-backend fa3 \
  --mm-attention-backend fa3  \
  --moe-runner-backend auto  \
  --sampling-backend flashinfer  \
  --linear-attn-backend flashinfer  \
  --tokenizer-backend fastokens  \
  --mem-fraction-static 0.75  \
  --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64  \
  --mamba-scheduler-strategy extra_buffer \
  --chunked-prefill-size -1  \
  --disable-overlap-schedule  \
  --watchdog-timeout 1200  \
  --soft-watchdog-timeout 1200  \
  --reasoning-parser qwen3 \
  --tool-call-parser qwen3_coder \  
  --decode-log-interval 1 > "${LOG_FILE}" 2>&1 &

Qwen3.5-122B-A10B​

本示例使用 1P1D 分离时部署，请准备2台机器。

prefill启动脚本 prefill_server.sh

SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0

export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEP_GEMM_BLOCK_M=256
# 如果模型为 BF16 数据类型则打开该环境变量
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true

export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600

export VLLM_PATCH_MUSA_CUSTOM_OPS=1
export SGLANG_DEEPEP_BF16_DISPATCH=1

export SGLANG_EXPERT_DISTRIBUTION_RECORDER_DIR="$PWD"/Prefill-qwen3.5

NODE_RANK=$1
PREFILL_IP="${2}:2345"
LOG_DIR=$3
MODEL_PATH=$4
HOST_IP=$5

WORLD_SIZE=1
WORK_HOME="$PWD"
# export SGLANG_TORCH_PROFILER_DIR=$WORK_HOME/P${NODE_RANK}_traces
# export SGLANG_EXPERT_DISTRIBUTION_RECORDER_DIR=$WORK_HOME/Prefill-qwen3.5
DEEP_EP_CONFIG="${SHARED_CONFIG_ROOT}/deepep/deepep.config"
EPLB_FILE="${SHARED_CONFIG_ROOT}/eplb/Qwen3.5-122B-A10B/prefill_eplb_qwen35.pt"

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SGLANG_PD_IB_RESOLVER="${SCRIPT_DIR}/resolve_sglang_pd_ib_devices.sh"
if [[ -f "$SGLANG_PD_IB_RESOLVER" ]]; then
    source "$SGLANG_PD_IB_RESOLVER"
else
    SGLANG_DISAGGREGATION_IB_DEVICES="${SGLANG_DISAGGREGATION_IB_DEVICES:-${SGLANG_IB_DEFAULT_DEVICES:-}}"
    export SGLANG_DISAGGREGATION_IB_DEVICES
    echo "Warning: ${SGLANG_PD_IB_RESOLVER} not found, use default IB devices"
    echo "IB_DEVICES=$SGLANG_DISAGGREGATION_IB_DEVICES"
fi

mkdir -p "${LOG_DIR}"

nohup  "$SGLANG_PYTHON"  -m sglang.launch_server \
--model-path ${MODEL_PATH} \
      --trust-remote-code \
      --disable-overlap-schedule \
      --disable-cuda-graph \
      --tp-size 8 \
      --dp-size 8 \
      --ep-size 8 \
      --pp-size 1 \
      --mem-fraction-static 0.8 \
      --attention-backend fa3 \
      --enable-cache-report \
      --mm-attention-backend fa3 \
      --moe-runner-backend auto \
      --sampling-backend flashinfer \
      --moe-dense-tp-size 1 \
      --enable-dp-lm-head \
      --enable-dp-attention \
      --moe-a2a-backend deepep \
      --deepep-mode normal \
      --dist-init-addr $PREFILL_IP \
      --nnodes ${WORLD_SIZE} \
      --node-rank ${NODE_RANK} \
      --deepep-config "$DEEP_EP_CONFIG" \
      --init-expert-location "$EPLB_FILE" \
      --chunked-prefill-size -1 \
      --max-running-requests 64 \
      --mamba-scheduler-strategy extra_buffer \
      --port 24586 \
      --host 0.0.0.0 \
      --speculative-algo NEXTN \
      --speculative-num-steps 1 \
      --speculative-eagle-topk 1 \
      --speculative-num-draft-tokens 2 \
      --load-balance-method round_robin \
      --disaggregation-mode prefill \
      --reasoning-parser qwen3 \
      --tool-call-parser qwen3_coder \
      --disaggregation-ib-device "$SGLANG_DISAGGREGATION_IB_DEVICES" > "${LOG_DIR}/P${NODE_RANK}_${HOST_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

decoder启动脚本 decode_server.sh

SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1


export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0

export MC_ENABLE_DEST_DEVICE_AFFINITY=1
export MC_TE_METRIC=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEP_GEMM_BLOCK_M=256
# 如果模型为 BF16 数据类型则打开该环境变量
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true

export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600

export SGLANG_DEEPEP_LL_USE_NVLINK=1
export SGLANG_DEEPEP_LL_DISABLE_RECV_HOOK=0
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128

export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export SGLANG_DISAGGREGATION_MAPPING_IB_DEVICE_TO_GPU=1
export SGLANG_BLACKWELL_OVERLAP_SHARED_EXPERTS_OUTSIDE_SBO=0
export SGLANG_DEEPEP_BF16_DISPATCH=1

WORK_HOME="$PWD"
export SGLANG_EXPERT_DISTRIBUTION_RECORDER_DIR=$WORK_HOME/Decode-qwen3.5

NODE_RANK=$1
DECODER_IP="${2}:4343"
LOG_DIR=$3
MODEL_PATH=$4
HOST_IP=$5

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SGLANG_PD_IB_RESOLVER="${SCRIPT_DIR}/resolve_sglang_pd_ib_devices.sh"
if [[ -f "$SGLANG_PD_IB_RESOLVER" ]]; then
    source "$SGLANG_PD_IB_RESOLVER"
else
    SGLANG_DISAGGREGATION_IB_DEVICES="${SGLANG_DISAGGREGATION_IB_DEVICES:-${SGLANG_IB_DEFAULT_DEVICES:-}}"
    export SGLANG_DISAGGREGATION_IB_DEVICES
    echo "Warning: ${SGLANG_PD_IB_RESOLVER} not found, use default IB devices"
    echo "IB_DEVICES=$SGLANG_DISAGGREGATION_IB_DEVICES"
fi

WORLD_SIZE=1
export SGLANG_TORCH_PROFILER_DIR=$WORK_HOME/D${NODE_RANK}_traces
EPLB_FILE="${SHARED_CONFIG_ROOT}/eplb/Qwen3.5-122B-A10B/decode_eplb_qwen35.pt"

nohup  "$SGLANG_PYTHON"  -m sglang.launch_server \
      --model-path ${MODEL_PATH} \
      --trust-remote-code \
      --disable-overlap-schedule \
      --cuda-graph-bs $(seq 1 32) \
      --tp-size 8 \
      --ep-size 8 \
      --dp-size 8 \
      --mem-fraction-static 0.65 \
      --attention-backend fa3 \
      --enable-cache-report \
      --mm-attention-backend fa3 \
      --moe-runner-backend auto \
      --enable-dp-lm-head \
      --moe-dense-tp-size 1 \
      --enable-dp-attention \
      --moe-a2a-backend deepep \
      --deepep-mode low_latency \
      --sampling-backend flashinfer \
      --dist-init-addr $DECODER_IP \
      --init-expert-location "$EPLB_FILE" \
      --nnodes ${WORLD_SIZE} \
      --node-rank ${NODE_RANK} \
      --port 23457 \
      --host 0.0.0.0 \
      --decode-log-interval 1 \
      --speculative-algo NEXTN \
      --speculative-num-steps 1 \
      --speculative-eagle-topk 1 \
      --speculative-num-draft-tokens 2 \
      --ep-num-redundant-experts 0 \
      --load-balance-method round_robin \
      --prefill-round-robin-balance \
      --disaggregation-mode decode \
      --reasoning-parser qwen3 \
      --tool-call-parser qwen3_coder \
      --disaggregation-ib-device "$SGLANG_DISAGGREGATION_IB_DEVICES" > "${LOG_DIR}/D${NODE_RANK}_${HOST_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

Router 启动脚本 router.sh

SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
PREFILL_IP="http://${1}:24586"
DECODE_IP="http://$2:23457"
LOG_DIR=$3

ulimit -n 65535
nohup "${SGLANG_PYTHON}" -m sglang_router.launch_router \
    --pd-disaggregation \
    --mini-lb \
    --prefill $PREFILL_IP \
    --decode $DECODE_IP \
    --host 0.0.0.0 \
    --request-timeout-secs 7200 \
    --port 22230 > "${LOG_DIR}/router_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

hostfile文件示例

192.168.100.101
192.168.100.102

一键运行脚本 run.sh

#!/bin/bash

  CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")

  PREFILL_SERVER_COUNT=${PREFILL_SERVER_COUNT:-1}
  DECODER_SERVER_COUNT=${DECODER_SERVER_COUNT:-1}
  WORLD_SIZE=$(( (PREFILL_SERVER_COUNT + DECODER_SERVER_COUNT) * 8 ))

  echo -e "\033[32mPREFILL_SERVER_COUNT: $PREFILL_SERVER_COUNT, DECODER_SERVER_COUNT: $DECODER_SERVER_COUNT, \033[0m"

  set -u
  WORK_HOME="$PWD"
  EXPNAME="P${PREFILL_SERVER_COUNT}_D${DECODER_SERVER_COUNT}_gpus${WORLD_SIZE}"
  MODEL_PATH=${MODEL_PATH:-"/data/models/Qwen3.5-122B-A10B-FP8/"}
  HOSTFILE=./hostfile
  LOG_DIR=$WORK_HOME/output/$EXPNAME/$CURRENT_TIME/
  PREFILL_SCRIPT_FILE="$WORK_HOME/prefill_server.sh"
  DECODER_SCRIPT_FILE="$WORK_HOME/decode_server.sh"
  ROUTER_SCRIPT_FILE="$WORK_HOME/router.sh"
  set +u

  SSH_PORT="${SSH_PORT:-62216}"
  REMOTE_SHARED_CONFIG_ROOT="${SHARED_CONFIG_ROOT:-/shared_configs}"
  REMOTE_DEEP_EP_CONFIG="${DEEP_EP_CONFIG:-${REMOTE_SHARED_CONFIG_ROOT}/deepep/deepep.config}"
  REMOTE_SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
  REMOTE_SGLANG_PYTHON="${SGLANG_PYTHON:-${REMOTE_SGLANG_VENV}/bin/python3}"

  REMOTE_ENV="SGLANG_VENV=$REMOTE_SGLANG_VENV SGLANG_PYTHON=$REMOTE_SGLANG_PYTHON SHARED_CONFIG_ROOT=$REMOTE_SHARED_CONFIG_ROOT DEEP_EP_CONFIG=$REMOTE_DEEP_EP_CONFIG"

  echo -e "\033[32mWORK_HOME: $WORK_HOME\033[0m"
  echo -e "\033[32mMODEL_PATH: $MODEL_PATH\033[0m"
  echo -e "\033[32mPREFILL_SCRIPT_FILE: $PREFILL_SCRIPT_FILE\033[0m"
  echo -e "\033[32mDECODER_SCRIPT_FILE: $DECODER_SCRIPT_FILE\033[0m"
  echo -e "\033[32mSHARED_CONFIG_ROOT: $REMOTE_SHARED_CONFIG_ROOT\033[0m"
  echo -e "\033[32mDEEP_EP_CONFIG: $REMOTE_DEEP_EP_CONFIG\033[0m"

  COUNT=0
  hostlist=$(grep -v '^#\|^$' "$HOSTFILE" | awk '{print $1}' | xargs)

  if [ -z "$hostlist" ]; then
    echo "Error: hostlist is empty. Please add IP addresses to the hostfile."
    exit 1
  fi

  PREFILL_MASTER_IP=$(head -n 1 "$HOSTFILE")
  DECODER_MASTER_IP=$(tail -n "$DECODER_SERVER_COUNT" "$HOSTFILE" | head -n 1)

  echo -e "\033[32mPREFILL_MASTER_IP: $PREFILL_MASTER_IP\033[0m"
  echo -e "\033[32mDECODER_MASTER_IP: $DECODER_MASTER_IP\033[0m"

  mkdir -p "$LOG_DIR"

  for host in ${hostlist[@]}; do
    echo "$host"

    if [[ $COUNT -lt $PREFILL_SERVER_COUNT ]]; then
      PREFILL_INDEX=$COUNT
      cmd="bash -c 'cd $WORK_HOME; $REMOTE_ENV bash $PREFILL_SCRIPT_FILE $PREFILL_INDEX $PREFILL_MASTER_IP $LOG_DIR $MODEL_PATH $host'"
      echo "$cmd"
      ssh -p "$SSH_PORT" -f -n "$host" "$cmd"
    else
      DECODER_INDEX=$((COUNT - PREFILL_SERVER_COUNT))
      cmd="bash -c 'cd $WORK_HOME; $REMOTE_ENV bash $DECODER_SCRIPT_FILE $DECODER_INDEX $DECODER_MASTER_IP $LOG_DIR $MODEL_PATH $host'"
      echo "$cmd"
      ssh -p "$SSH_PORT" -f -n "$host" "$cmd"
    fi

    ((COUNT++))
  done

  cmd="bash -c 'cd $WORK_HOME; $REMOTE_ENV bash $ROUTER_SCRIPT_FILE $PREFILL_MASTER_IP $DECODER_MASTER_IP $LOG_DIR'"
  echo "$cmd"
  ssh -p "$SSH_PORT" -f -n "$PREFILL_MASTER_IP" "$cmd"

  echo -e "Main log file: \033[34m$LOG_DIR\033[0m"

Qwen3.5-397B-A17B-FP8​

本示例使用 1P4D 分离式部署，请准备5台机器。DeepEP 的配置文件内容参考 DeepEP 配置文件

prefill启动脚本 prefill_server.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

export TP_SOCKET_IFNAME=bond0
export GLOO_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_PROTOS=2
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0

export MC_TE_METRIC=1
export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_DEEP_GEMM_BLOCK_M=256
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true
export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_DISAGGREGATION_MAPPING_IB_DEVICE_TO_GPU=1
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SGLANG_IB_TARGET_COUNT=8
SGLANG_IB_REPEAT_PER_DEVICE=2
SGLANG_PD_IB_RESOLVER="${SCRIPT_DIR}/resolve_sglang_pd_ib_devices.sh"
if [[ -f "$SGLANG_PD_IB_RESOLVER" ]]; then
    source "$SGLANG_PD_IB_RESOLVER"
else
    SGLANG_DISAGGREGATION_IB_DEVICES="${SGLANG_DISAGGREGATION_IB_DEVICES:-${SGLANG_IB_DEFAULT_DEVICES:-}}"
    export SGLANG_DISAGGREGATION_IB_DEVICES
    echo "Warning: ${SGLANG_PD_IB_RESOLVER} not found, use default IB devices"
    echo "IB_DEVICES=$SGLANG_DISAGGREGATION_IB_DEVICES"
fi
export SGLANG_TORCH_PROFILER_DIR=/data/workspace/Qwen3.5-397B-A17B-FP8/1P2D/1P2D/traces
export SGLANG_EXPERT_DISTRIBUTION_RECORDER_DIR=/data/workspace/Qwen3.5-397B-A17B-FP8/1P2D/1P2D/traces

SGLANG_PORT="${SGLANG_PORT:-30233}"
MASTER_IP=$1
NODE_RANK=$2
NNODES=$3
WORLD_SIZE=$NNODES
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="${4:-$WORK_HOME/output/$CURRENT_TIME/}"
MODEL_PATH="${5:-/data/models/Qwen3.5-397B-A17B-FP8}"
mkdir -p "$LOG_DIR"
SHARED_DIR="${SGLANG_SHARED_DIR:-${SCRIPT_DIR}/shared}"
if [[ -z "${SGLANG_SHARED_DIR:-}" && ! -d "$SHARED_DIR" ]]; then
    for shared_candidate in "${SCRIPT_DIR}/../shared" "${SCRIPT_DIR}/../../shared" "${SCRIPT_DIR}/../../../shared" "${SCRIPT_DIR}/../../../../shared"; do
        if [[ -d "$shared_candidate" ]]; then
            SHARED_DIR="$(cd "$shared_candidate" && pwd)"
            break
        fi
    done
fi

DEEP_EP_CONFIG="/data/workspace/deepep/deepep.config"

echo "DEEP_EP_CONFIG=$DEEP_EP_CONFIG"


# 清理缓存
rm -rf ~/.triton/cache
rm -rf /tmp/*

nohup "$SGLANG_PYTHON" -m sglang.launch_server \
    --disable-piecewise-cuda-graph \
    --model $MODEL_PATH \
    --trust-remote-code \
    --disable-cuda-graph \
    --disable-overlap-schedule \
    --tp-size 8 \
    --ep-size 8 \
    --dp-size 8 \
    --pp-size 1 \
    --moe-dense-tp-size 1 \
    --enable-dp-lm-head \
    --enable-dp-attention \
    --moe-a2a-backend deepep \
    --deepep-mode normal \
    --mem-fraction-static 0.8 \
    --attention-backend fa3 \
    --linear-attn-backend flashinfer \
    --moe-runner-backend auto \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --tokenizer-backend fastokens \
    --dist-init-addr ${MASTER_IP}:5403 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size -1 \
    --mamba-scheduler-strategy extra_buffer \
    --enable-cache-report \
    --deepep-config "$DEEP_EP_CONFIG" \
    --schedule-conservativeness 1 \
    --max-running-requests 64 \
    --port ${SGLANG_PORT} \
    --host "${SGLANG_HOST}" \
    --load-balance-method round_robin \
    --disaggregation-mode prefill \
    --disaggregation-ib-device "$SGLANG_DISAGGREGATION_IB_DEVICES" \
    --reasoning-parser qwen3 \
    --tool-call-parser qwen3_coder > "${LOG_DIR}/prefill_server_${NODE_RANK}.log" 2>&1 &

decoder启动脚本 decode_server.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

export TP_SOCKET_IFNAME=bond0
export GLOO_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_PROTOS=2
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0

export MC_TE_METRIC=1
export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_DEEPEP_LL_USE_NVLINK=1
export SGLANG_DEEPEP_LL_DISABLE_RECV_HOOK=0
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true
export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_DISAGGREGATION_MAPPING_IB_DEVICE_TO_GPU=1
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SGLANG_IB_TARGET_COUNT=8
SGLANG_IB_REPEAT_PER_DEVICE=2
SGLANG_PD_IB_RESOLVER="${SCRIPT_DIR}/resolve_sglang_pd_ib_devices.sh"
if [[ -f "$SGLANG_PD_IB_RESOLVER" ]]; then
    source "$SGLANG_PD_IB_RESOLVER"
else
    SGLANG_DISAGGREGATION_IB_DEVICES="${SGLANG_DISAGGREGATION_IB_DEVICES:-${SGLANG_IB_DEFAULT_DEVICES:-}}"
    export SGLANG_DISAGGREGATION_IB_DEVICES
    echo "Warning: ${SGLANG_PD_IB_RESOLVER} not found, use default IB devices"
    echo "IB_DEVICES=$SGLANG_DISAGGREGATION_IB_DEVICES"
fi
export SGLANG_TORCH_PROFILER_DIR=/data/workspace/Qwen3.5-397B-A17B-FP8/1P2D/1P2D/traces
export SGLANG_EXPERT_DISTRIBUTION_RECORDER_DIR=/data/workspace/Qwen3.5-397B-A17B-FP8/1P2D/1P2D/traces
export SGLANG_BLACKWELL_OVERLAP_SHARED_EXPERTS_OUTSIDE_SBO=0

SGLANG_PORT="${SGLANG_PORT:-30233}"
MASTER_IP=$1
NODE_RANK=$2
NNODES=$3
WORLD_SIZE=$NNODES
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="${4:-$WORK_HOME/output/$CURRENT_TIME/}"
MODEL_PATH="${5:-/data/models/Qwen3.5-397B-A17B-FP8}"
mkdir -p "$LOG_DIR"

# 清理缓存
rm -rf ~/.triton/cache
rm -rf /tmp/*

nohup "$SGLANG_PYTHON" -m sglang.launch_server \
    --model $MODEL_PATH \
    --trust-remote-code \
    --disable-overlap-schedule \
    --cuda-graph-bs $(seq 1 32) \
    --disable-piecewise-cuda-graph \
    --tp-size $((WORLD_SIZE * 8)) \
    --ep-size $((WORLD_SIZE * 8)) \
    --dp-size $((WORLD_SIZE * 8)) \
    --enable-dp-lm-head \
    --moe-dense-tp-size 1 \
    --enable-dp-attention \
    --moe-a2a-backend deepep \
    --deepep-mode low_latency \
    --mem-fraction-static 0.65 \
    --attention-backend fa3 \
    --linear-attn-backend flashinfer \
    --moe-runner-backend auto \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --tokenizer-backend fastokens \
    --max-running-requests 256 \
    --dist-init-addr ${MASTER_IP}:5533 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size -1 \
    --speculative-algorithm NEXTN \
    --speculative-num-steps 1 \
    --speculative-eagle-topk 1 \
    --speculative-num-draft-tokens 2 \
    --port ${SGLANG_PORT} \
    --host "${SGLANG_HOST}" \
    --decode-log-interval 1 \
    --prefill-round-robin-balance \
    --disaggregation-mode decode \
    --disaggregation-ib-device "$SGLANG_DISAGGREGATION_IB_DEVICES" \
    --reasoning-parser qwen3 \
    --tool-call-parser qwen3_coder > "${LOG_DIR}/decode_server_${NODE_RANK}.log" 2>&1 &

Router 启动脚本 router.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"

SGLANG_PREFILL_PORT="${SGLANG_PREFILL_PORT:-30233}"
SGLANG_DECODE_PORT="${SGLANG_DECODE_PORT:-30233}"
SGLANG_ROUTER_PORT="${SGLANG_ROUTER_PORT:-31100}"
PREFILL_IP="http://${1}:$SGLANG_PREFILL_PORT"
DECODE_IP="http://${2}:$SGLANG_DECODE_PORT"
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="${3:-$WORK_HOME/output/$CURRENT_TIME/}"
mkdir -p "$LOG_DIR"

lsof -ti:"$SGLANG_ROUTER_PORT" | xargs -r kill -9

nohup "$SGLANG_PYTHON" -m sglang_router.launch_router \
    --pd-disaggregation \
    --prefill "$PREFILL_IP" \
    --decode "$DECODE_IP" \
    --host "${SGLANG_HOST}" \
    --mini-lb \
    --request-timeout-secs 7200 \
    --worker-startup-timeout-secs 600 \
    --port "$SGLANG_ROUTER_PORT" > "${LOG_DIR}/router.log" 2>&1 &

hostfile文件示例

192.168.100.101
192.168.100.102
192.168.100.103
192.168.100.104
192.168.100.105

一键运行脚本 run.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_PORT="${SGLANG_PORT:-30233}"
SGLANG_PREFILL_PORT="${SGLANG_PREFILL_PORT:-$SGLANG_PORT}"
SGLANG_DECODE_PORT="${SGLANG_DECODE_PORT:-$SGLANG_PORT}"
SGLANG_ROUTER_PORT="${SGLANG_ROUTER_PORT:-31100}"
SSH_PORT="${SSH_PORT:-62216}"

WORKSPACE=$(pwd)
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="$WORK_HOME/output/$CURRENT_TIME/"
PREFILL_SERVER_COUNT=${PREFILL_SERVER_COUNT:-1}
DECODER_SERVER_COUNT=${DECODER_SERVER_COUNT:-4}
PREFILL_NNODES=${PREFILL_NNODES:-$PREFILL_SERVER_COUNT}
DECODER_NNODES=${DECODER_NNODES:-$DECODER_SERVER_COUNT}
REQUIRED_NODES=$((PREFILL_SERVER_COUNT + DECODER_SERVER_COUNT))
MODEL_PATH="${MODEL_PATH:-/data/models/Qwen3.5-397B-A17B-FP8}"
mkdir -p "$LOG_DIR"

# 检查 hostfile 是否存在
if [ ! -f "$WORKSPACE/hostfile" ]; then
    echo "Error: $WORKSPACE/hostfile not found!"
    exit 1
fi

# 读取 hostfile 到数组
mapfile -t hosts < "$WORKSPACE/hostfile"

# 检查是否读取到足够的节点
if [[ ${#hosts[@]} -lt "$REQUIRED_NODES" ]]; then
    echo "Error: hostfile should contain at least $REQUIRED_NODES nodes"
    exit 1
fi

echo "Using hosts:"
printf '%s\n' "${hosts[@]}"
echo "MODEL_PATH: $MODEL_PATH"
echo "PREFILL_SERVER_COUNT: $PREFILL_SERVER_COUNT"
echo "DECODER_SERVER_COUNT: $DECODER_SERVER_COUNT"

PREFILL_MASTER_IP=${hosts[0]}
DECODER_MASTER_IP=${hosts[$PREFILL_SERVER_COUNT]}

REMOTE_SHARED_CONFIG_ROOT="${SHARED_CONFIG_ROOT:-/data/workspace}"
REMOTE_DEEP_EP_CONFIG="${DEEP_EP_CONFIG:-${REMOTE_SHARED_CONFIG_ROOT}/deepep/deepep.config}"
REMOTE_SMART_TEST_HOST_RUN_DIR="${SMART_TEST_HOST_RUN_DIR:-}"
REMOTE_ENV="SGLANG_VENV=$SGLANG_VENV SGLANG_PYTHON=$SGLANG_PYTHON SGLANG_PORT=$SGLANG_PORT SGLANG_PREFILL_PORT=$SGLANG_PREFILL_PORT SGLANG_DECODE_PORT=$SGLANG_DECODE_PORT SGLANG_ROUTER_PORT=$SGLANG_ROUTER_PORT SMART_TEST_HOST_RUN_DIR=$REMOTE_SMART_TEST_HOST_RUN_DIR SHARED_CONFIG_ROOT=$REMOTE_SHARED_CONFIG_ROOT DEEP_EP_CONFIG=$REMOTE_DEEP_EP_CONFIG"
echo "PREFILL_MASTER_IP: $PREFILL_MASTER_IP"
echo "DECODER_MASTER_IP: $DECODER_MASTER_IP"

# prefill
for ((prefill_rank = 0; prefill_rank < PREFILL_SERVER_COUNT; prefill_rank++)); do
    prefill_host=${hosts[$prefill_rank]}
    ssh -p "$SSH_PORT" "$prefill_host" "cd $WORKSPACE && SGLANG_HOST=$prefill_host $REMOTE_ENV bash prefill_server.sh $PREFILL_MASTER_IP $prefill_rank $PREFILL_NNODES $LOG_DIR $MODEL_PATH" &
done

# decode
for ((decode_rank = 0; decode_rank < DECODER_SERVER_COUNT; decode_rank++)); do
    decode_host_index=$((PREFILL_SERVER_COUNT + decode_rank))
    decode_host=${hosts[$decode_host_index]}
    ssh -p "$SSH_PORT" "$decode_host" "cd $WORKSPACE && SGLANG_HOST=$decode_host $REMOTE_ENV bash decode_server.sh $DECODER_MASTER_IP $decode_rank $DECODER_NNODES $LOG_DIR $MODEL_PATH" &
done

# router - 在第一个节点上启动 router
ssh -p "$SSH_PORT" "$PREFILL_MASTER_IP" "cd $WORKSPACE && $REMOTE_ENV bash router.sh $PREFILL_MASTER_IP $DECODER_MASTER_IP $LOG_DIR" &

echo "All services started in background"
echo "All logs are stored in $LOG_DIR"

版权所有 2024 摩尔线程，本文档受国际版权法保护。

摩尔线程和摩尔线程徽标是摩尔线程智能科技 (北京)有限责任公司的注册商标。

免责声明

本文档提供有关摩尔线程产品的信息。本文档并未授权任何知识产权的许可，并未以明示或暗示，或以禁止反言或其他方式授予任何知识产权许可。