Qwen3 VL 模型部署

本章节包含 Qwen3 VL 系列模型的部署脚本，脚本涉及的参数说明、PD 分离一键部署说明见 快速开始。

• 模型权重下载。

• 参考 环境准备 进行环境检查、启动容器。

• 容器镜像地址参考 版本发布信息-镜像地址。

• 参考 环境准备-RDMA 网卡检测 中的脚本，请其保存为 resolve_sglang_pd_ib_devices.sh，与 SGLang 服务启动脚本放在同一路径下。

Qwen3VL-2/4/8B

本示例使用 1卡（TP1）部署 Qwen3-VL-2B-Instruct。 对于 Qwen3-VL-4B-Instruct 和 Qwen3-VL-8B-Instruct，修改 MODEL_PATH 环境变量和 served-model-name 参数即可。

启动脚本 start_server.sh

SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export SGLANG_TORCH_PROFILER_DIR=/tmp/traces
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
export MUSA_LAUNCH_BLOCKING="${MUSA_LAUNCH_BLOCKING:-0}"
export TORCH_MCCL_HEARTBEAT_TIMEOUT_SEC="${TORCH_MCCL_HEARTBEAT_TIMEOUT_SEC:-3600}"
export SGLANG_MUSA_DISABLE_TILELANG_DEEPGEMM_PREPROCESS="${SGLANG_MUSA_DISABLE_TILELANG_DEEPGEMM_PREPROCESS:-1}"
export VLLM_PATCH_MUSA_CUSTOM_OPS="${VLLM_PATCH_MUSA_CUSTOM_OPS:-1}"
export SGLANG_USE_CUDA_IPC_TRANSPORT="${SGLANG_USE_CUDA_IPC_TRANSPORT:-1}"
export SGLANG_USE_IPC_POOL_HANDLE_CACHE="${SGLANG_USE_IPC_POOL_HANDLE_CACHE:-1}"
# export SGLANG_MOE_CONFIG_DIR=/sgl-workspace/sglang/python/sglang/srt/layers/moe/moe_runner/triton_utils/

# pkill -f "sgl.*"
# sleep 3

CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
SGLANG_PORT="${SGLANG_PORT:-42244}"
PORT_2=5123
MASTER_IP=127.0.0.1
NODE_RANK=0
WORLD_SIZE=1
MODEL_PATH=/data/models/Qwen3-VL-2B-Instruct-FP8/
LOG_DIR="$WORK_HOME/output/$CURRENT_TIME/"
LOG_FILE="${LOG_DIR}/sglang_$(date '+%Y%m%d_%H%M%S').log"
LATEST_LOG_LINK="$WORK_HOME/output/latest"

mkdir -p "$LOG_DIR"
rm -f "$LATEST_LOG_LINK"
ln -s "$CURRENT_TIME/$(basename "$LOG_FILE")" "$LATEST_LOG_LINK"

nohup "$SGLANG_PYTHON" -m sglang.launch_server \
    --model-path "$MODEL_PATH" \
    --trust-remote-code \
    --cuda-graph-max-bs 256 \
    --disable-overlap-schedule \
    --tp-size 1 \
    --pp-size 1 \
    --mem-fraction-static 0.8 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --linear-attn-backend flashinfer \
    --moe-runner-backend auto \
    --tokenizer-backend fastokens \
    --sampling-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:$PORT_2 \
    --nnodes 1 \
    --node-rank 0 \
    --chunked-prefill-size -1 \
    --port ${SGLANG_PORT} \
    --host "${SGLANG_HOST}" \
    --decode-log-interval 1 > "$LOG_FILE" 2>&1 &

Qwen3VL-32B

本示例使用 4卡（TP4）部署。

启动脚本 start_server.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export SGLANG_TORCH_PROFILER_DIR=/tmp/traces
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
export LD_LIBRARY_PATH=/usr/local/musa/lib:${LD_LIBRARY_PATH}
export SGLANG_USE_CUDA_IPC_TRANSPORT="${SGLANG_USE_CUDA_IPC_TRANSPORT:-1}"
export SGLANG_USE_IPC_POOL_HANDLE_CACHE="${SGLANG_USE_IPC_POOL_HANDLE_CACHE:-1}"

SGLANG_PORT="${SGLANG_PORT:-31000}"
MASTER_IP=127.0.0.1
NODE_RANK=0
WORLD_SIZE=1
MODEL_PATH=/data/models/Qwen3-VL-32B-Instruct-FP8/
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="$WORK_HOME/output/$CURRENT_TIME/"
LOG_FILE="${LOG_DIR}/sglang_$(date '+%Y%m%d_%H%M%S').log"
LATEST_LOG_LINK="$WORK_HOME/output/latest"

mkdir -p "$LOG_DIR"
rm -f "$LATEST_LOG_LINK"
ln -s "$CURRENT_TIME/$(basename "$LOG_FILE")" "$LATEST_LOG_LINK"

nohup "$SGLANG_PYTHON" -m sglang.launch_server \
    --model ${MODEL_PATH} \
    --trust-remote-code \
    --cuda-graph-max-bs 256 \
    --disable-overlap-schedule \
    --tp-size 4 \
    --dp-size 1 \
    --mem-fraction-static 0.8 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --linear-attn-backend flashinfer \
    --tokenizer-backend fastokens \
    --sampling-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:50122 \
    --nnodes 1 \
    --node-rank 0 \
    --chunked-prefill-size -1 \
    --port ${SGLANG_PORT} \
    --host "${SGLANG_HOST}" \
    --decode-log-interval 1 \
    --reasoning-parser qwen3 \
    --tool-call-parser qwen > "$LOG_FILE" 2>&1 &

Qwen3-VL-30B-A3B-Instruct

本示例使用 2卡（TP2）部署。

启动脚本 start_server.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export SGLANG_TORCH_PROFILER_DIR=/tmp/traces
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
export MUSA_LAUNCH_BLOCKING="${MUSA_LAUNCH_BLOCKING:-0}"
export TORCH_MCCL_HEARTBEAT_TIMEOUT_SEC="${TORCH_MCCL_HEARTBEAT_TIMEOUT_SEC:-3600}"
export SGLANG_MUSA_DISABLE_TILELANG_DEEPGEMM_PREPROCESS="${SGLANG_MUSA_DISABLE_TILELANG_DEEPGEMM_PREPROCESS:-1}"
export VLLM_PATCH_MUSA_CUSTOM_OPS="${VLLM_PATCH_MUSA_CUSTOM_OPS:-1}"
export SGLANG_USE_CUDA_IPC_TRANSPORT="${SGLANG_USE_CUDA_IPC_TRANSPORT:-1}"
export SGLANG_USE_IPC_POOL_HANDLE_CACHE="${SGLANG_USE_IPC_POOL_HANDLE_CACHE:-1}"
# export SGLANG_MOE_CONFIG_DIR=/sgl-workspace/sglang/python/sglang/srt/layers/moe/moe_runner/triton_utils/

# pkill -f "sgl.*"
# sleep 3

CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
SGLANG_PORT="${SGLANG_PORT:-42244}"
PORT_2=5123
MASTER_IP=127.0.0.1
NODE_RANK=0
WORLD_SIZE=1
MODEL_PATH=/data/models/Qwen3-VL-30B-A3B-Instruct-FP8/
LOG_DIR="$WORK_HOME/output/$CURRENT_TIME/"
LOG_FILE="${LOG_DIR}/sglang_$(date '+%Y%m%d_%H%M%S').log"
LATEST_LOG_LINK="$WORK_HOME/output/latest"

mkdir -p "$LOG_DIR"
rm -f "$LATEST_LOG_LINK"
ln -s "$CURRENT_TIME/$(basename "$LOG_FILE")" "$LATEST_LOG_LINK"

nohup "$SGLANG_PYTHON" -m sglang.launch_server \
    --model-path "$MODEL_PATH" \
    --trust-remote-code \
    --cuda-graph-max-bs 512 \
    --disable-overlap-schedule \
    --tp-size 2 \
    --ep-size 2 \
    --pp-size 1 \
    --mem-fraction-static 0.8 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --linear-attn-backend flashinfer \
    --moe-runner-backend auto \
    --tokenizer-backend fastokens \
    --sampling-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:$PORT_2 \
    --nnodes 1 \
    --node-rank 0 \
    --max-prefill-tokens 4096 \
    --chunked-prefill-size -1 \
    --port ${SGLANG_PORT} \
    --host "${SGLANG_HOST}" \
    --decode-log-interval 1 \
    --reasoning-parser qwen3 \
    --tool-call-parser qwen > "$LOG_FILE" 2>&1 &

Qwen3-VL-235B-A22B-Instruct

本示例部署方式为1P2D，请准备三台机器。模型使用FP8，如需运行BF16版本，修改 MODEL_PATH 环境变量和 served-model-name 参数即可。

prefill启动脚本 prefill_server.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export MCCL_SOCKET_IFNAME=bond0

export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEP_GEMM_BLOCK_M=256
# 如果模型为 BF16 数据类型则打开该环境变量
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true

export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600

export VLLM_PATCH_MUSA_CUSTOM_OPS=1

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SGLANG_PREFILL_PORT="${SGLANG_PREFILL_PORT:-24586}"
SGLANG_IB_TARGET_COUNT=8
SGLANG_IB_REPEAT_PER_DEVICE=2
SGLANG_PD_IB_RESOLVER="${SCRIPT_DIR}/resolve_sglang_pd_ib_devices.sh"
if [[ -f "$SGLANG_PD_IB_RESOLVER" ]]; then
    source "$SGLANG_PD_IB_RESOLVER"
else
    SGLANG_DISAGGREGATION_IB_DEVICES="${SGLANG_DISAGGREGATION_IB_DEVICES:-${SGLANG_IB_DEFAULT_DEVICES:-}}"
    export SGLANG_DISAGGREGATION_IB_DEVICES
    echo "Warning: ${SGLANG_PD_IB_RESOLVER} not found, use default IB devices"
    echo "IB_DEVICES=$SGLANG_DISAGGREGATION_IB_DEVICES"
fi

NODE_RANK=$1
PREFILL_IP="${2}:2345"
LOG_DIR=$3
MODEL_PATH=$4
HOST_IP=$5
NNODES=$6
WORLD_SIZE=$NNODES
WORK_HOME="$PWD"
export SGLANG_TORCH_PROFILER_DIR=$WORK_HOME/P${NODE_RANK}_traces

mkdir -p "${LOG_DIR}"

nohup "$SGLANG_PYTHON" -m sglang.launch_server \
--model-path ${MODEL_PATH} \
      --trust-remote-code \
      --disable-overlap-schedule \
      --disable-cuda-graph \
      --tp-size 8 \
      --dp-size 8 \
      --ep-size 8 \
      --pp-size 1 \
      --mem-fraction-static 0.85 \
      --attention-backend fa3 \
      --enable-cache-report \
      --mm-attention-backend fa3 \
      --moe-runner-backend auto \
      --sampling-backend flashinfer \
      --moe-dense-tp-size 1 \
      --enable-dp-lm-head \
      --enable-dp-attention \
      --moe-a2a-backend deepep \
      --deepep-mode normal \
      --dist-init-addr $PREFILL_IP \
      --nnodes ${WORLD_SIZE} \
      --node-rank ${NODE_RANK} \
      --chunked-prefill-size -1 \
      --max-running-requests 64 \
      --port "$SGLANG_PREFILL_PORT" \
      --host "${SGLANG_HOST}" \
      --load-balance-method round_robin \
      --disaggregation-mode prefill \
      --disaggregation-ib-device "$SGLANG_DISAGGREGATION_IB_DEVICES" \
      --reasoning-parser qwen3 \
      --tool-call-parser qwen > "${LOG_DIR}/P${NODE_RANK}_${HOST_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

decoder启动脚本 decoder_server.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export MCCL_SOCKET_IFNAME=bond0

export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEP_GEMM_BLOCK_M=256
# 如果模型为 BF16 数据类型则打开该环境变量
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true

export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600

export SGLANG_DEEPEP_LL_USE_NVLINK=1
export SGLANG_DEEPEP_LL_DISABLE_RECV_HOOK=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128

export VLLM_PATCH_MUSA_CUSTOM_OPS=1

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SGLANG_DECODE_PORT="${SGLANG_DECODE_PORT:-23457}"
SGLANG_IB_TARGET_COUNT=8
SGLANG_IB_REPEAT_PER_DEVICE=2
SGLANG_PD_IB_RESOLVER="${SCRIPT_DIR}/resolve_sglang_pd_ib_devices.sh"
if [[ -f "$SGLANG_PD_IB_RESOLVER" ]]; then
    source "$SGLANG_PD_IB_RESOLVER"
else
    SGLANG_DISAGGREGATION_IB_DEVICES="${SGLANG_DISAGGREGATION_IB_DEVICES:-${SGLANG_IB_DEFAULT_DEVICES:-}}"
    export SGLANG_DISAGGREGATION_IB_DEVICES
    echo "Warning: ${SGLANG_PD_IB_RESOLVER} not found, use default IB devices"
    echo "IB_DEVICES=$SGLANG_DISAGGREGATION_IB_DEVICES"
fi

NODE_RANK=$1
DECODER_IP="${2}:4343"
LOG_DIR=$3
MODEL_PATH=$4
HOST_IP=$5
NNODES=$6
WORLD_SIZE=$NNODES
WORK_HOME="$PWD"
export SGLANG_TORCH_PROFILER_DIR=$WORK_HOME/D${NODE_RANK}_traces

mkdir -p "${LOG_DIR}"

nohup "$SGLANG_PYTHON" -m sglang.launch_server \
      --model-path ${MODEL_PATH} \
      --trust-remote-code \
      --disable-overlap-schedule \
      --cuda-graph-bs $(seq 1 32) \
      --tp-size 8 \
      --ep-size 8 \
      --dp-size 8 \
      --mem-fraction-static 0.76 \
      --attention-backend fa3 \
      --enable-cache-report \
      --mm-attention-backend fa3 \
      --moe-runner-backend auto \
      --enable-dp-lm-head \
      --moe-dense-tp-size 1 \
      --enable-dp-attention \
      --moe-a2a-backend deepep \
      --deepep-mode low_latency \
      --sampling-backend flashinfer \
      --dist-init-addr $DECODER_IP \
      --nnodes ${WORLD_SIZE} \
      --node-rank ${NODE_RANK} \
      --port "$SGLANG_DECODE_PORT" \
      --host "${SGLANG_HOST}" \
      --ep-num-redundant-experts 0 \
      --load-balance-method round_robin \
      --prefill-round-robin-balance \
      --disaggregation-mode decode \
      --disaggregation-ib-device "$SGLANG_DISAGGREGATION_IB_DEVICES" \
      --reasoning-parser qwen3 \
      --tool-call-parser qwen > "${LOG_DIR}/D${NODE_RANK}_${HOST_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

Router 启动脚本 router.sh

export PATH=/root/.virtualenvs/sglang-default/bin/:$PATH
PREFILL_IP="http://${1}:24586"
DECODE_IP="http://$2:23457"
LOG_DIR=$3

ulimit -n 65535
nohup python -m sglang_router.launch_router \
    --pd-disaggregation \
    --mini-lb \
    --prefill $PREFILL_IP \
    --decode $DECODE_IP \
    --host 0.0.0.0 \
    --request-timeout-secs 7200 \
    --port 22230 > "${LOG_DIR}/router_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

hostfile文件示例

192.168.100.101
192.168.100.102
192.168.100.103

一键运行脚本 run.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"
SSH_PORT="${SSH_PORT:-62216}"

CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")

PREFILL_SERVER_COUNT=${PREFILL_SERVER_COUNT:-1}
DECODER_SERVER_COUNT=${DECODER_SERVER_COUNT:-2}
WORLD_SIZE=$(((PREFILL_SERVER_COUNT + DECODER_SERVER_COUNT) * 8))

echo -e "\033[32mPREFILL_SERVER_COUNT: $PREFILL_SERVER_COUNT, DECODER_SERVER_COUNT: $DECODER_SERVER_COUNT, \033[0m"

set -u
WORK_HOME="$PWD"
EXPNAME="P${PREFILL_SERVER_COUNT}_D${DECODER_SERVER_COUNT}_gpus${WORLD_SIZE}"
MODEL_PATH=${MODEL_PATH:-"/data/models/Qwen3-VL-235B-A22B-Instruct-FP8/"}
HOSTFILE=./hostfile
LOG_DIR="$WORK_HOME/output/$CURRENT_TIME/"
PREFILL_SCRIPT_FILE="$WORK_HOME/prefill_server.sh"
DECODER_SCRIPT_FILE="$WORK_HOME/decoder_server.sh"
ROUTER_SCRIPT_FILE="$WORK_HOME/router.sh"
set +u

echo -e "\033[32mWORK_HOME: $WORK_HOME\033[0m"
echo -e "\033[32mMODEL_PATH: $MODEL_PATH\033[0m"
echo -e "\033[32mPREFILL_SCRIPT_FILE: $PREFILL_SCRIPT_FILE\033[0m"
echo -e "\033[32mDECODER_SCRIPT_FILE: $DECODER_SCRIPT_FILE\033[0m"

COUNT=0
hostlist=$(grep -v '^#\|^$' $HOSTFILE | awk '{print $1}' | xargs)

if [ -z "$hostlist" ]; then
  echo "Error: hostlist is empty. Please add IP addresses to the hostfile."
  exit 1
fi

PREFILL_MASTER_IP=$(head -n 1 $HOSTFILE)
DECODER_MASTER_IP=$(tail -n $DECODER_SERVER_COUNT $HOSTFILE | head -n1)
echo -e "\033[32mPREFILL_MASTER_IP: $PREFILL_MASTER_IP\033[0m"
echo -e "\033[32mDECODER_MASTER_IP: $DECODER_MASTER_IP\033[0m"

mkdir -p $LOG_DIR
for host in ${hostlist[@]}; do
  echo $host
  if [[ $COUNT -lt $PREFILL_SERVER_COUNT ]]; then
    PREFILL_INDEX=$COUNT
    cmd="bash -c 'cd $WORK_HOME; SGLANG_VENV=$SGLANG_VENV SGLANG_PYTHON=$SGLANG_PYTHON bash $PREFILL_SCRIPT_FILE $PREFILL_INDEX $PREFILL_MASTER_IP $LOG_DIR $MODEL_PATH $host $PREFILL_SERVER_COUNT'"
    echo $cmd
    ssh -p "$SSH_PORT" -f -n $host $cmd
  else
    DECODER_INDEX=$((COUNT - PREFILL_SERVER_COUNT))
    cmd="bash -c 'cd $WORK_HOME; SGLANG_VENV=$SGLANG_VENV SGLANG_PYTHON=$SGLANG_PYTHON bash $DECODER_SCRIPT_FILE $DECODER_INDEX $DECODER_MASTER_IP $LOG_DIR $MODEL_PATH $host $DECODER_SERVER_COUNT'"
    echo $cmd
    ssh -p "$SSH_PORT" -f -n $host $cmd
  fi
  ((COUNT++))
done
cmd="bash -c 'cd $WORK_HOME; SGLANG_VENV=$SGLANG_VENV SGLANG_PYTHON=$SGLANG_PYTHON bash $ROUTER_SCRIPT_FILE $PREFILL_MASTER_IP $DECODER_MASTER_IP $LOG_DIR'"
echo $cmd
ssh -p "$SSH_PORT" -f -n $PREFILL_MASTER_IP $cmd
echo -e "Main log file: \033[34m$LOG_DIR\033[0m"