Gemma-4-26B-A4B-it 模型部署

本章节包含 Gemma-4 系列模型的部署脚本及说明。

• 模型权重下载。

• 参考 环境准备 进行环境检查、启动容器。

• 容器镜像地址参考 版本发布信息-镜像地址。

Gemma-4-26B-A4B-it

8卡部署 （tp=8） 示例如下

启动脚本 start_server.sh

#!/usr/bin/env bash
set -euo pipefail

cd "$(dirname "$0")"

MODEL_NAME="gemma-4-26B-A4B-it"

MODEL_PATH="${MODEL_PATH:-/models/gemma-4-26B-A4B-it}"
TOKENIZER_PATH="${TOKENIZER_PATH:-$MODEL_PATH}"
HOST="${HOST:-0.0.0.0}"
PORT="${PORT:-22231}"
GEMMA4_VENV="${GEMMA4_VENV:-/root/.virtualenvs/gemma4}"

[[ -x "$GEMMA4_VENV/bin/python" ]] || {
  echo "gemma4 venv python not executable: $GEMMA4_VENV/bin/python" >&2
  exit 2
}

export VIRTUAL_ENV="$GEMMA4_VENV"
export PATH="$VIRTUAL_ENV/bin:$PATH"
PYTHON="${PYTHON:-$VIRTUAL_ENV/bin/python}"

[[ -d "$MODEL_PATH" ]] || {
  echo "MODEL_PATH does not exist: $MODEL_PATH" >&2
  exit 2
}
[[ -d "$TOKENIZER_PATH" ]] || {
  echo "TOKENIZER_PATH does not exist: $TOKENIZER_PATH" >&2
  exit 2
}
[[ -x "$PYTHON" ]] || {
  echo "PYTHON not executable: $PYTHON" >&2
  exit 2
}

TP_SIZE="${TP_SIZE:-8}"
MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.80}"

export LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu:/usr/local/musa/lib:${LD_LIBRARY_PATH:-}"
export MUSA_ENABLE_LLC_OPT="${MUSA_ENABLE_LLC_OPT:-1}"
export VLLM_PATCH_MUSA_CUSTOM_OPS="${VLLM_PATCH_MUSA_CUSTOM_OPS:-1}"
export SGLANG_ENABLE_JIT_DEEPGEMM="${SGLANG_ENABLE_JIT_DEEPGEMM:-1}"
export SGLANG_MUSA_CONTIG_DEEPGEMM_TOKEN_THRESHOLD="${SGLANG_MUSA_CONTIG_DEEPGEMM_TOKEN_THRESHOLD:-99999999}"

CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="${LOG_DIR:-$WORK_HOME/output/$CURRENT_TIME}"
LOG_FILE="${LOG_FILE:-$LOG_DIR/sglang_$(date '+%Y%m%d_%H%M%S').log}"
LATEST_LOG_LINK="$WORK_HOME/output/latest"
HEALTH_WAIT_SECONDS="${HEALTH_WAIT_SECONDS:-900}"

mkdir -p "$LOG_DIR"
mkdir -p "$(dirname "$LATEST_LOG_LINK")"
rm -f "$LATEST_LOG_LINK"
ln -s "$CURRENT_TIME/$(basename "$LOG_FILE")" "$LATEST_LOG_LINK"

if command -v lsof >/dev/null 2>&1; then
  lsof -ti:"$PORT" | xargs -r kill -9 || true
fi
pkill -9 -f "[s]glang.launch_server.*--port[[:space:]]+${PORT}" || true
pkill -9 -f "[s]glang.launch_server.*${MODEL_NAME}" || true
sleep 3

cd /sgl-workspace/sglang
"$PYTHON" -m pip install transformers==5.12.0

cmd=(
  "$PYTHON" -m sglang.launch_server
  --model-path "$MODEL_PATH"
  --trust-remote-code
  --mem-fraction-static "${MEM_FRACTION_STATIC}"
  --tp-size "$TP_SIZE"
  --random-seed 314159 
  --attention-backend fa3 
  --mm-attention-backend fa3 
  --page-size 64 
  --cuda-graph-bs 1 2 3 4 8 12 16 
  --disable-piecewise-cuda-graph 
  --max-prefill-tokens 65536 
  --skip-server-warmup 
  --chunked-prefill-size -1 
  --prefill-max-requests 1 
  --tokenizer-worker-num 1 
  --moe-runner-backend auto
  --watchdog-timeout 2400
  --soft-watchdog-timeout 2400
  --host "$HOST"
  --port "$PORT"
  --decode-log-interval 1
)


printf '%q ' "${cmd[@]}" >"${LOG_FILE}.cmd"
printf '\n' >>"${LOG_FILE}.cmd"

echo "starting ${MODEL_NAME} long-sequence profile on ${HOST}:${PORT}"
echo "python: ${PYTHON}"
echo "venv: ${VIRTUAL_ENV}"
echo "log: ${LOG_FILE}"

if [[ "${FOREGROUND:-0}" == "1" ]]; then
  exec "${cmd[@]}"
fi

nohup "${cmd[@]}" >"$LOG_FILE" 2>&1 &
server_pid=$!
echo "$server_pid" >"${LOG_FILE}.pid"
echo "pid: $(cat "${LOG_FILE}.pid")"

if [[ "${WAIT_FOR_HEALTH:-0}" != "1" ]]; then
  sleep "${STARTUP_PROBE_SECONDS:-5}"
  if ! kill -0 "$server_pid" 2>/dev/null; then
    echo "server process exited after launch; tailing log" >&2
    tail -200 "$LOG_FILE" >&2 || true
    exit 1
  fi
  echo "health check skipped; process alive pid=${server_pid}"
  exit 0
fi

for _ in $(seq 1 "$HEALTH_WAIT_SECONDS"); do
  if ! kill -0 "$server_pid" 2>/dev/null; then
    echo "server process exited before health check passed; tailing log" >&2
    tail -200 "$LOG_FILE" >&2 || true
    exit 1
  fi

  curl -sf "http://127.0.0.1:${PORT}/health" >/dev/null && {
    echo "health OK on port ${PORT}"
    exit 0
  }

  sleep 1
done

tail -200 "$LOG_FILE" >&2 || true
exit 1