Minimax 模型部署

本章节包含 Minimax 模型的部署脚本及说明。

• 模型权重下载。

• 参考 环境准备 进行环境检查、启动容器。

• 容器镜像地址参考 版本发布信息-镜像地址。

Minimax-M3-FP8 单机部署

单机部署 （tp=8） 示例如下。

启动脚本 start_server.sh

#!/usr/bin/env bash
set -uo pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

MODEL="${MODEL:-/data/models/MiniMax-M3-FP8}"
DRAFT="${DRAFT:-/data/models/MiniMax-M3-EAGLE3}"
PORT="${PORT:-30000}"
LOG="${LOG:-./server.eagle3.log}"
CTX="${CTX:-131072}"
MEMFRAC="${MEMFRAC:-0.7}"
CG_MAX_BS="${CG_MAX_BS:-16}"
MAXREQ="${MAXREQ:-16}"

SGLANG_VENV="${SGLANG_VENV:-/root/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-$SGLANG_VENV/bin/python3}"
LOG_DIR="${LOG_DIR:-${SMART_TEST_SERVICE_OUTPUT_DIR:-${RUN_DIR:-/run_dir}/logs/service/sglang}}"
LOG_FILE="${LOG:-$LOG_DIR/server.eagle3.log}"
mkdir -p "$LOG_DIR"

[[ -e "$MODEL" ]] || echo "[launch] WARN: MODEL=$MODEL not found - set MODEL=..." >&2
[[ -e "$DRAFT" ]] || echo "[launch] WARN: DRAFT=$DRAFT not found - set DRAFT=... (EAGLE3 draft weights)" >&2

# --- MUSA runtime libs -------------------------------------------------------
export LD_LIBRARY_PATH=/usr/local/musa/lib:/usr/local/musa/lib64:/usr/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}

# --- keep torch_musa autoload ON --------------------------------------------
unset TORCH_DEVICE_BACKEND_AUTOLOAD || true

# --- per-host JIT caches ----------------------------------------------------
export MATE_WORKSPACE_BASE="${MATE_WORKSPACE_BASE:-$HOME/.cache/minimax-m3-fp8/mate_workspace}"
export SGLANG_DG_CACHE_DIR="${SGLANG_DG_CACHE_DIR:-$HOME/.cache/minimax-m3-fp8/deep_gemm}"
mkdir -p "$MATE_WORKSPACE_BASE"
mkdir -p "$SGLANG_DG_CACHE_DIR"
mkdir -p "$(dirname "$LOG_FILE")"

# --- TP token id sync -------------------------------------------------------
export SYNC_TOKEN_IDS_ACROSS_TP=1
export SGLANG_ENABLE_SPEC_V2=1

echo "[launch] MiniMax-M3-FP8 EAGLE3 model=$MODEL draft=$DRAFT port=$PORT ctx=$CTX mem=$MEMFRAC" >&2

nohup "$SGLANG_PYTHON" -m sglang.launch_server \
  --model-path "$MODEL" \
  --trust-remote-code \
  --tp 8 \
  --dtype bfloat16 \
  --quantization fp8 \
  --attention-backend fa3 \
  --page-size 128 \
  --mem-fraction-static "$MEMFRAC" \
  --context-length "$CTX" \
  --max-running-requests "$MAXREQ" \
  --max-prefill-tokens 8192 \
  --chunked-prefill-size 8192 \
  --speculative-algorithm EAGLE3 \
  --speculative-draft-model-path "$DRAFT" \
  --speculative-num-steps 3 \
  --speculative-eagle-topk 1 \
  --speculative-num-draft-tokens 4 \
  --cuda-graph-max-bs "$CG_MAX_BS" \
  --host 0.0.0.0 \
  --port "$PORT" \
  --reasoning-parser auto \
  --tool-call-parser auto \
  --decode-log-interval 5 \
  --watchdog-timeout 600 \
  "$@" > "$LOG_FILE" 2>&1 &

Minimax-M3-FP8 PD分离部署

PD分离部署示例如下，使用1个 Prefill 节点和1个 Decode节点。

• PD分离脚本涉及的参数说明和一键运行说明见 快速开始。

• 参考 环境准备-RDMA 网卡检测 中的脚本，请其保存为 resolve_sglang_pd_ib_devices.sh，与 SGLang 服务启动脚本放在同一路径下。

Prefill启动

prefill启动脚本 prefill_server.sh

#!/bin/bash
# MiniMax-M3-FP8 PD prefill, one node TP8, chunked prefill 32768, prefix cache off.
# Args: NODE_RANK MASTER_IP LOG_DIR MODEL_PATH HOST_IP

set -uo pipefail

SGLANG_VENV="${SGLANG_VENV:-${HOME}/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
if [[ ! -x "$SGLANG_PYTHON" ]]; then
  SGLANG_PYTHON=python3
fi

export PYTHONUNBUFFERED=1
export LD_LIBRARY_PATH="/usr/local/musa/lib:/usr/local/musa/lib64:/usr/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}"
export MUSA_VISIBLE_DEVICES="${MUSA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
export MUSA_LAUNCH_BLOCKING="${MUSA_LAUNCH_BLOCKING:-0}"
export MUSA_ENABLE_LLC_OPT="${MUSA_ENABLE_LLC_OPT:-1}"
export SYNC_TOKEN_IDS_ACROSS_TP=1
export SGLANG_MUSA_M3_MSA=1
export SGLANG_MUSA_M3_MSA_BACKEND=mate
export SGLANG_MUSA_M3_MSA_INDEXER_BACKEND=mate
export SGLANG_MUSA_M3_MSA_ATTN_BACKEND=mate
export SGLANG_MUSA_M3_MSA_FP8_ATTN=1
export SGLANG_MOE_PADDING=0
export SGLANG_MUSA_M3_OPROJ_ROWS="${SGLANG_MUSA_M3_OPROJ_ROWS:-1}"
export MATE_MUSA_ARCH_LIST="${MATE_MUSA_ARCH_LIST:-3.1}"
export SGLANG_DISAGGREGATION_QUEUE_SIZE="${SGLANG_DISAGGREGATION_QUEUE_SIZE:-8}"
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE="${SGLANG_DISAGGREGATION_THREAD_POOL_SIZE:-16}"
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT="${SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT:-6000}"
export MCCL_IB_GID_INDEX="${MCCL_IB_GID_INDEX:-3}"
export MCCL_NET_SHARED_BUFFERS="${MCCL_NET_SHARED_BUFFERS:-0}"
export MCCL_PROTOS="${MCCL_PROTOS:-2}"
export GLOO_SOCKET_IFNAME="${GLOO_SOCKET_IFNAME:-bond0}"
export TP_SOCKET_IFNAME="${TP_SOCKET_IFNAME:-bond0}"
export MCCL_SOCKET_IFNAME="${MCCL_SOCKET_IFNAME:-bond0}"

JIT_CACHE_DIR="${JIT_CACHE_DIR:-${HOME}/.cache/minimax-m3-fp8}"
export MATE_WORKSPACE_BASE="${MATE_WORKSPACE_BASE:-$JIT_CACHE_DIR/mate}"
export SGLANG_DG_CACHE_DIR="${SGLANG_DG_CACHE_DIR:-$JIT_CACHE_DIR/deep_gemm}"
export TRITON_CACHE_DIR="${TRITON_CACHE_DIR:-$JIT_CACHE_DIR/triton}"
mkdir -p "$MATE_WORKSPACE_BASE" "$SGLANG_DG_CACHE_DIR" "$TRITON_CACHE_DIR"

SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-MiniMax-M3-FP8}"
CONTEXT_LENGTH="${CONTEXT_LENGTH:-270336}"
MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-64}"
BOOTSTRAP_PORT="${BOOTSTRAP_PORT:-9999}"
WATCHDOG_TIMEOUT="${WATCHDOG_TIMEOUT:-1800}"
PREFILL_PORT="${PREFILL_PORT:-24586}"
DECODE_PORT="${DECODE_PORT:-23457}"
ROUTER_PORT="${ROUTER_PORT:-31000}"
DECODE_GRAPH_BS="${DECODE_GRAPH_BS:-1 2 4 8 12 16 24 32 48 64}"

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
if [[ -f "${SCRIPT_DIR}/resolve_sglang_pd_ib_devices.sh" ]]; then
  # shellcheck disable=SC1091
  source "${SCRIPT_DIR}/resolve_sglang_pd_ib_devices.sh"
else
  SGLANG_DISAGGREGATION_IB_DEVICES="${SGLANG_DISAGGREGATION_IB_DEVICES:-mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_6,mlx5_7}"
  export SGLANG_DISAGGREGATION_IB_DEVICES
  echo "IB_DEVICES=$SGLANG_DISAGGREGATION_IB_DEVICES"
fi


NODE_RANK="${1:?node rank}"
MASTER_IP="${2:?prefill master ip}"
LOG_DIR="${3:?log dir}"
MODEL_PATH="${4:-${MODEL_PATH:-/models/MiniMax-M3-FP8}}"
HOST_IP="${5:-}"
DIST_PORT="${PREFILL_DIST_PORT:-29771}"
CHUNK="${PREFILL_CHUNK:-32768}"
MEM="${PREFILL_MEM:-0.80}"

mkdir -p "$LOG_DIR"
LOG_FILE="${LOG_DIR}/P${NODE_RANK}_${HOST_IP}_$(date '+%Y%m%d_%H%M%S').log"
echo "prefill starting model=$MODEL_PATH host=$HOST_IP rank=$NODE_RANK port=$PREFILL_PORT log=$LOG_FILE"
cat > /usr/local/lib/python3.10/dist-packages/sitecustomize.py << 'SITE'
import sglang.srt.layers.attention.minimax_sparse_backend as backend

_original = backend._mate_prefill_attention_backend

def _short_prefill_uses_triton(selected, *, num_tokens, block_size):
    if selected == "mate" and num_tokens <= block_size:
        return "triton"
    return _original(selected, num_tokens=num_tokens, block_size=block_size)

backend._mate_prefill_attention_backend = _short_prefill_uses_triton
SITE

nohup "$SGLANG_PYTHON" -m sglang.launch_server \
  --model-path "$MODEL_PATH" \
  --served-model-name "$SERVED_MODEL_NAME" \
  --trust-remote-code \
  --dtype bfloat16 \
  --quantization fp8 \
  --kv-cache-dtype fp8_e4m3 \
  --attention-backend fa3 \
  --page-size 128 \
  --tp 8 \
  --mem-fraction-static "$MEM" \
  --context-length "$CONTEXT_LENGTH" \
  --max-running-requests "$MAX_RUNNING_REQUESTS" \
  --chunked-prefill-size "$CHUNK" \
  --max-prefill-tokens "$CHUNK" \
  --sampling-backend pytorch \
  --watchdog-timeout "$WATCHDOG_TIMEOUT" \
  --disable-radix-cache \
  --disaggregation-mode prefill \
  --disaggregation-transfer-backend mooncake \
  --disaggregation-bootstrap-port "$BOOTSTRAP_PORT" \
  --disaggregation-ib-device "$SGLANG_DISAGGREGATION_IB_DEVICES" \
  --disable-cuda-graph \
  --disable-overlap-schedule \
  --dist-init-addr "${MASTER_IP}:${DIST_PORT}" \
  --nnodes 1 \
  --node-rank "$NODE_RANK" \
  --host 0.0.0.0 \
  --port "$PREFILL_PORT" \
  > "$LOG_FILE" 2>&1 &

echo "prefill pid=$! log=$LOG_FILE"

Decode启动

decoder启动脚本 decode_server.sh

#!/bin/bash
# MiniMax-M3-FP8 PD decode, one node TP8, mem-fraction 0.88.
# Args: NODE_RANK MASTER_IP LOG_DIR MODEL_PATH HOST_IP
set -uo pipefail

SGLANG_VENV="${SGLANG_VENV:-${HOME}/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
if [[ ! -x "$SGLANG_PYTHON" ]]; then
  SGLANG_PYTHON=python3
fi

export PYTHONUNBUFFERED=1
export LD_LIBRARY_PATH="/usr/local/musa/lib:/usr/local/musa/lib64:/usr/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}"
export MUSA_VISIBLE_DEVICES="${MUSA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
export MUSA_LAUNCH_BLOCKING="${MUSA_LAUNCH_BLOCKING:-0}"
export MUSA_ENABLE_LLC_OPT="${MUSA_ENABLE_LLC_OPT:-1}"
export SYNC_TOKEN_IDS_ACROSS_TP=1
export SGLANG_MUSA_M3_MSA=1
export SGLANG_MUSA_M3_MSA_BACKEND=mate
export SGLANG_MUSA_M3_MSA_INDEXER_BACKEND=mate
export SGLANG_MUSA_M3_MSA_ATTN_BACKEND=mate
export SGLANG_MUSA_M3_MSA_FP8_ATTN=1
export SGLANG_MOE_PADDING=0
export SGLANG_MUSA_M3_OPROJ_ROWS="${SGLANG_MUSA_M3_OPROJ_ROWS:-1}"
export MATE_MUSA_ARCH_LIST="${MATE_MUSA_ARCH_LIST:-3.1}"
export SGLANG_DISAGGREGATION_QUEUE_SIZE="${SGLANG_DISAGGREGATION_QUEUE_SIZE:-8}"
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE="${SGLANG_DISAGGREGATION_THREAD_POOL_SIZE:-16}"
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT="${SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT:-6000}"
export MCCL_IB_GID_INDEX="${MCCL_IB_GID_INDEX:-3}"
export MCCL_NET_SHARED_BUFFERS="${MCCL_NET_SHARED_BUFFERS:-0}"
export MCCL_PROTOS="${MCCL_PROTOS:-2}"
export GLOO_SOCKET_IFNAME="${GLOO_SOCKET_IFNAME:-bond0}"
export TP_SOCKET_IFNAME="${TP_SOCKET_IFNAME:-bond0}"
export MCCL_SOCKET_IFNAME="${MCCL_SOCKET_IFNAME:-bond0}"

JIT_CACHE_DIR="${JIT_CACHE_DIR:-${HOME}/.cache/minimax-m3-fp8}"
export MATE_WORKSPACE_BASE="${MATE_WORKSPACE_BASE:-$JIT_CACHE_DIR/mate}"
export SGLANG_DG_CACHE_DIR="${SGLANG_DG_CACHE_DIR:-$JIT_CACHE_DIR/deep_gemm}"
export TRITON_CACHE_DIR="${TRITON_CACHE_DIR:-$JIT_CACHE_DIR/triton}"
mkdir -p "$MATE_WORKSPACE_BASE" "$SGLANG_DG_CACHE_DIR" "$TRITON_CACHE_DIR"

SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-MiniMax-M3-FP8}"
CONTEXT_LENGTH="${CONTEXT_LENGTH:-270336}"
MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-64}"
BOOTSTRAP_PORT="${BOOTSTRAP_PORT:-9999}"
WATCHDOG_TIMEOUT="${WATCHDOG_TIMEOUT:-1800}"
PREFILL_PORT="${PREFILL_PORT:-24586}"
DECODE_PORT="${DECODE_PORT:-23457}"
ROUTER_PORT="${ROUTER_PORT:-31000}"
DECODE_GRAPH_BS="${DECODE_GRAPH_BS:-1 2 4 8 12 16 24 32 48 64}"

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
if [[ -f "${SCRIPT_DIR}/resolve_sglang_pd_ib_devices.sh" ]]; then
  # shellcheck disable=SC1091
  source "${SCRIPT_DIR}/resolve_sglang_pd_ib_devices.sh"
else
  SGLANG_DISAGGREGATION_IB_DEVICES="${SGLANG_DISAGGREGATION_IB_DEVICES:-mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_6,mlx5_7}"
  export SGLANG_DISAGGREGATION_IB_DEVICES
  echo "IB_DEVICES=$SGLANG_DISAGGREGATION_IB_DEVICES"
fi

NODE_RANK="${1:?node rank}"
MASTER_IP="${2:?decode master ip}"
LOG_DIR="${3:?log dir}"
MODEL_PATH="${4:-${MODEL_PATH:-/models/MiniMax-M3-FP8}}"
HOST_IP="${5:-}"
DIST_PORT="${DECODE_DIST_PORT:-29772}"
CHUNK="${DECODE_CHUNK:-4096}"
MEM="${DECODE_MEM:-0.88}"

mkdir -p "$LOG_DIR"
LOG_FILE="${LOG_DIR}/D${NODE_RANK}_${HOST_IP}_$(date '+%Y%m%d_%H%M%S').log"
echo "decode starting model=$MODEL_PATH host=$HOST_IP rank=$NODE_RANK port=$DECODE_PORT log=$LOG_FILE"

# shellcheck disable=SC2086
nohup "$SGLANG_PYTHON" -m sglang.launch_server \
  --model-path "$MODEL_PATH" \
  --served-model-name "$SERVED_MODEL_NAME" \
  --trust-remote-code \
  --dtype bfloat16 \
  --quantization fp8 \
  --kv-cache-dtype fp8_e4m3 \
  --attention-backend fa3 \
  --page-size 128 \
  --tp 8 \
  --mem-fraction-static "$MEM" \
  --context-length "$CONTEXT_LENGTH" \
  --max-running-requests "$MAX_RUNNING_REQUESTS" \
  --chunked-prefill-size "$CHUNK" \
  --max-prefill-tokens "$CHUNK" \
  --sampling-backend pytorch \
  --watchdog-timeout "$WATCHDOG_TIMEOUT" \
  --disable-radix-cache \
  --disaggregation-mode decode \
  --disaggregation-transfer-backend mooncake \
  --disaggregation-ib-device "$SGLANG_DISAGGREGATION_IB_DEVICES" \
  --disable-piecewise-cuda-graph \
  --cuda-graph-bs-decode $DECODE_GRAPH_BS \
  --dist-init-addr "${MASTER_IP}:${DIST_PORT}" \
  --nnodes 1 \
  --node-rank "$NODE_RANK" \
  --host 0.0.0.0 \
  --port "$DECODE_PORT" \
  --decode-log-interval 1 \
  > "$LOG_FILE" 2>&1 &

echo "decode pid=$! log=$LOG_FILE"

Router启动

Router 启动脚本 router.sh

#!/bin/bash
# Args: PREFILL_IP DECODE_IP LOG_DIR
set -uo pipefail
SGLANG_VENV="${SGLANG_VENV:-${HOME}/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
if [[ ! -x "$SGLANG_PYTHON" ]]; then
  SGLANG_PYTHON=python3
fi

PREFILL_IP="${1:?prefill ip}"
DECODE_IP="${2:?decode ip}"
LOG_DIR="${3:?log dir}"
PREFILL_PORT="${SGLANG_PREFILL_PORT:-${PREFILL_PORT:-24586}}"
DECODE_PORT="${SGLANG_DECODE_PORT:-${DECODE_PORT:-23457}}"
ROUTER_PORT="${SGLANG_ROUTER_PORT:-${ROUTER_PORT:-31000}}"
BOOTSTRAP_PORT="${BOOTSTRAP_PORT:-9999}"
mkdir -p "$LOG_DIR"
LOG_FILE="${LOG_DIR}/router_$(date '+%Y%m%d_%H%M%S').log"
ulimit -n 65535 || true

echo "router starting prefill=http://${PREFILL_IP}:${PREFILL_PORT} bootstrap=${BOOTSTRAP_PORT} decode=http://${DECODE_IP}:${DECODE_PORT} port=${ROUTER_PORT}"
nohup "${SGLANG_PYTHON}" -m sglang_router.launch_router \
    --pd-disaggregation \
    --mini-lb \
    --prefill "http://${PREFILL_IP}:${PREFILL_PORT}" "${BOOTSTRAP_PORT}" \
    --decode "http://${DECODE_IP}:${DECODE_PORT}" \
    --host 0.0.0.0 \
    --port "${ROUTER_PORT}" \
    --policy random \
    --request-timeout-secs 14400 \
    > "$LOG_FILE" 2>&1 < /dev/null &
ROUTER_PID=$!
echo "router pid=${ROUTER_PID} log=$LOG_FILE port=${ROUTER_PORT}"

一键启动所有服务

一键运行脚本 run.sh

#!/bin/bash
# SmartTest entry for MiniMax-M3-FP8 1P1D.
# hostfile order: first host is prefill, second host is decode. Router runs on prefill.
set -uo pipefail

PREFILL_SERVER_COUNT=${PREFILL_SERVER_COUNT:-1}
DECODER_SERVER_COUNT=${DECODER_SERVER_COUNT:-1}

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
WORK_HOME="${WORK_HOME:-$PWD}"
MODEL_PATH="${MODEL_PATH:-/models/MiniMax-M3-FP8}"
HOSTFILE="${HOSTFILE:-${SMART_TEST_HOSTFILE:-./hostfile}}"
SSH_PORT="${SSH_PORT:-62216}"
LOG_DIR="${LOG_DIR:-${SMART_TEST_SERVICE_OUTPUT_DIR:-$WORK_HOME/output}}"
PREFILL_SCRIPT_FILE="${SCRIPT_DIR}/prefill_server.sh"
DECODER_SCRIPT_FILE="${SCRIPT_DIR}/decode_server.sh"
ROUTER_SCRIPT_FILE="${SCRIPT_DIR}/router.sh"
SGLANG_VENV="${SGLANG_VENV:-${HOME}/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"

mkdir -p "$LOG_DIR"
exec > >(tee -a "${LOG_DIR}/launch.log") 2>&1

echo "PREFILL_SERVER_COUNT=$PREFILL_SERVER_COUNT DECODER_SERVER_COUNT=$DECODER_SERVER_COUNT"
echo "WORK_HOME=$WORK_HOME"
echo "MODEL_PATH=$MODEL_PATH"
echo "HOSTFILE=$HOSTFILE"
echo "LOG_DIR=$LOG_DIR"
echo "SSH_PORT=$SSH_PORT"

if [[ ! -f "$HOSTFILE" ]]; then
  echo "Error: hostfile not found: $HOSTFILE"
  exit 1
fi

mapfile -t HOSTS < <(grep -v '^#\|^$' "$HOSTFILE" | awk '{print $1}')
NEED=$((PREFILL_SERVER_COUNT + DECODER_SERVER_COUNT))
if [[ ${#HOSTS[@]} -lt $NEED ]]; then
  echo "Error: hostfile has ${#HOSTS[@]} hosts, 1P1D needs $NEED"
  exit 1
fi

PREFILL_MASTER_IP="${HOSTS[0]}"
DECODER_MASTER_IP="${HOSTS[$PREFILL_SERVER_COUNT]}"
echo "PREFILL_MASTER_IP=$PREFILL_MASTER_IP"
echo "DECODER_MASTER_IP=$DECODER_MASTER_IP"

REMOTE_ENV="MODEL_PATH=$(printf '%q' "$MODEL_PATH") SGLANG_VENV=$(printf '%q' "$SGLANG_VENV") SGLANG_PYTHON=$(printf '%q' "$SGLANG_PYTHON") PREFILL_PORT=${PREFILL_PORT:-24586} DECODE_PORT=${DECODE_PORT:-23457} ROUTER_PORT=${ROUTER_PORT:-31000} BOOTSTRAP_PORT=${BOOTSTRAP_PORT:-9999}"

COUNT=0
for host in "${HOSTS[@]}"; do
  if [[ $COUNT -ge $NEED ]]; then
    break
  fi
  if [[ $COUNT -lt $PREFILL_SERVER_COUNT ]]; then
    INDEX=$COUNT
    cmd="bash -c 'cd $(printf '%q' "$WORK_HOME"); $REMOTE_ENV bash $(printf '%q' "$PREFILL_SCRIPT_FILE") $INDEX $PREFILL_MASTER_IP $(printf '%q' "$LOG_DIR") $(printf '%q' "$MODEL_PATH") $host'"
  else
    INDEX=$((COUNT - PREFILL_SERVER_COUNT))
    cmd="bash -c 'cd $(printf '%q' "$WORK_HOME"); $REMOTE_ENV bash $(printf '%q' "$DECODER_SCRIPT_FILE") $INDEX $DECODER_MASTER_IP $(printf '%q' "$LOG_DIR") $(printf '%q' "$MODEL_PATH") $host'"
  fi
  echo "$cmd"
  ssh -o StrictHostKeyChecking=no -p "$SSH_PORT" -f -n "$host" "$cmd"
  COUNT=$((COUNT + 1))
done

echo "Main log dir: $LOG_DIR"
echo "waiting for prefill and decode before starting router"

python3 - "$LOG_DIR" << 'PY'
import pathlib
import sys
import time

log_dir = pathlib.Path(sys.argv[1])
markers = {
    "prefill": ("P*.log", "The server is fired up and ready to roll"),
    "decode": ("D*.log", "The server is fired up and ready to roll"),
}
deadline = time.time() + 3600
seen = set()

def newest(pattern):
    files = [path for path in log_dir.glob(pattern) if path.is_file()]
    if not files:
        return None
    return max(files, key=lambda path: path.stat().st_mtime)

while time.time() < deadline and len(seen) < len(markers):
    for role, (pattern, text) in markers.items():
        if role in seen:
            continue
        path = newest(pattern)
        if path is None:
            continue
        content = path.read_text(errors="ignore")[-200000:]
        if text in content:
            print(role + " server up source=" + str(path), flush=True)
            seen.add(role)
    time.sleep(5)

if len(seen) < len(markers):
    missing = ",".join(sorted(set(markers) - seen))
    print("startup wait timed out missing=" + missing, flush=True)
    raise SystemExit(1)
print("prefill and decode ready", flush=True)
PY

ROUTER_PORT_VALUE="${ROUTER_PORT:-31000}"
ROUTER_CMD="bash -c 'cd $(printf '%q' "$WORK_HOME"); $REMOTE_ENV bash $(printf '%q' "$ROUTER_SCRIPT_FILE") $PREFILL_MASTER_IP $DECODER_MASTER_IP $(printf '%q' "$LOG_DIR")'"
echo "$ROUTER_CMD"
ssh -o StrictHostKeyChecking=no -p "$SSH_PORT" -f -n "$PREFILL_MASTER_IP" "$ROUTER_CMD"

python3 - "$LOG_DIR" "$ROUTER_PORT_VALUE" << 'PY'
import pathlib
import sys
import time

log_dir = pathlib.Path(sys.argv[1])
router_port = sys.argv[2]
deadline = time.time() + 180
pattern = "router_*.log"
ready_text = "Uvicorn running on http://0.0.0.0:" + router_port

def newest():
    files = [path for path in log_dir.glob(pattern) if path.is_file()]
    if not files:
        return None
    return max(files, key=lambda path: path.stat().st_mtime)

while time.time() < deadline:
    path = newest()
    if path is not None:
        content = path.read_text(errors="ignore")[-200000:]
        if ready_text in content:
            print("router ready source=" + str(path), flush=True)
            raise SystemExit(0)
        if "Error starting router" in content or "Traceback" in content:
            print("router failed source=" + str(path), flush=True)
            raise SystemExit(1)
    time.sleep(2)
print("router startup wait timed out", flush=True)
raise SystemExit(1)
PY