Qwen3 Dense 模型部署

本章节包含 Qwen3 Dense 系列模型的部署脚本及说明。支持 BF16 和 FP8 两种权重，部署时注意替换权重文件。

• 模型权重下载。

• 参考 环境准备 进行环境检查、启动容器。

• 容器镜像地址参考 版本发布信息-镜像地址。

Qwen3-8B

投机采样使用了 Draft Model，模型链接（Qwen3-8B_eagle3）

启动脚本 start_server.sh

#!/usr/bin/env bash
set -euo pipefail

SGLANG_VENV="${SGLANG_VENV:-/root/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
if [[ -f "${SGLANG_VENV}/bin/activate" ]]; then
  # shellcheck disable=SC1091
  source "${SGLANG_VENV}/bin/activate"
fi

export PATH="${SGLANG_VENV}/bin:${PATH}"
export LD_LIBRARY_PATH="/usr/local/musa/lib:/usr/local/openmpi/lib:/usr/local/musa/mudnn/lib:/usr/local/mtshmem/lib:${LD_LIBRARY_PATH:-}"
export MUSA_VISIBLE_DEVICES="${MUSA_VISIBLE_DEVICES:-0}"
export MTHREADS_VISIBLE_DEVICES="${MTHREADS_VISIBLE_DEVICES:-0}"
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_USE_DEEPGEMM_BMM=0
export SGLANG_MOE_PADDING=1
export SGLANG_USE_MUSA_FUSED_KERNEL=1

WORK_HOME="${WORK_HOME:-$PWD}"
MODEL_PATH="${MODEL_PATH:-/data/models/Qwen3-8B-FP8}"
MODEL_NAME="${MODEL_NAME:-Qwen3-8B-FP8}"
PORT="${PORT:-${SGLANG_PORT:-42251}}"
DIST_PORT="${DIST_PORT:-1264}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"
LOG_DIR="${LOG_DIR:-${WORK_HOME}/logs/service}"
SPEC_MODEL_PATH="${SPEC_MODEL_PATH:-/data/models/Qwen3-8B_eagle3}"
ENABLE_SPECULATIVE="${ENABLE_SPECULATIVE:-1}"
TOKENIZER_BACKEND="${TOKENIZER_BACKEND:-fastokens}"

SGLANG_ENABLE_SPECULATIVE="${SGLANG_ENABLE_SPECULATIVE:-${ENABLE_SPECULATIVE}}"
export SGLANG_ENABLE_SPEC_V2="${SGLANG_ENABLE_SPEC_V2:-0}"
mkdir -p "${LOG_DIR}" "${WORK_HOME}/traces"
TP_SIZE="${TP_SIZE:-1}"

MASTER_IP="$(hostname -I | awk '{print $1}')"
LOG_FILE="${LOG_DIR}/sglang_${MODEL_NAME}_$(date +%Y%m%d_%H%M%S).log"

SPEC_ARGS=()
if [[ "${SGLANG_ENABLE_SPECULATIVE}" == "1" || "${SGLANG_ENABLE_SPECULATIVE}" == "true" ]]; then
  export SGLANG_ENABLE_SPEC_V2=1
  SPEC_ARGS=(
    --speculative-draft-model-path "${SPEC_MODEL_PATH}"
    --speculative-algorithm EAGLE3
    --speculative-num-steps 3
    --speculative-eagle-topk 1
    --speculative-num-draft-tokens 4
  )
fi

TOKENIZER_ARGS=()
if [[ -n "${TOKENIZER_BACKEND}" ]]; then
  TOKENIZER_ARGS=(--tokenizer-backend "${TOKENIZER_BACKEND}")
fi

nohup "${SGLANG_PYTHON}" -m sglang.launch_server \
  --model-path "${MODEL_PATH}" \
  --served-model-name "${MODEL_NAME}" \
  --trust-remote-code \
  --tp-size "${TP_SIZE}" \
  --pp-size 1 \
  --host "${SGLANG_HOST}" \
  --port "${PORT}" \
  --dist-init-addr "${MASTER_IP}:${DIST_PORT}" \
  --attention-backend fa3 \
  --mm-attention-backend fa3 \
  --linear-attn-backend flashinfer \
  --sampling-backend flashinfer \
  "${TOKENIZER_ARGS[@]}" \
  --mem-fraction-static 0.82 \
  --cuda-graph-max-bs $(seq 1 256) \
  --disable-piecewise-cuda-graph \
  --chunked-prefill-size -1 \
  --max-running-requests 256 \
  --decode-log-interval 1 \
  "${SPEC_ARGS[@]}" \
  --reasoning-parser qwen3 \
  --tool-call-parser qwen25 \
  > "${LOG_FILE}" 2>&1 &

echo "SGLang ${MODEL_NAME} started, pid=$!, log=${LOG_FILE}"

Qwen3-14B

启动脚本 start_server.sh

#!/usr/bin/env bash
set -euo pipefail

SGLANG_VENV="${SGLANG_VENV:-/root/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
if [[ -f "${SGLANG_VENV}/bin/activate" ]]; then
  # shellcheck disable=SC1091
  source "${SGLANG_VENV}/bin/activate"
fi

export PATH="${SGLANG_VENV}/bin:${PATH}"
export LD_LIBRARY_PATH="/usr/local/musa/lib:/usr/local/openmpi/lib:/usr/local/musa/mudnn/lib:/usr/local/mtshmem/lib:${LD_LIBRARY_PATH:-}"
export MUSA_VISIBLE_DEVICES="${MUSA_VISIBLE_DEVICES:-0}"
export MTHREADS_VISIBLE_DEVICES="${MTHREADS_VISIBLE_DEVICES:-0}"
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_USE_DEEPGEMM_BMM=0
export SGLANG_MOE_PADDING=1
export SGLANG_USE_MUSA_FUSED_KERNEL=1

WORK_HOME="${WORK_HOME:-$PWD}"
MODEL_PATH="${MODEL_PATH:-/data/models/Qwen3-14B}"
MODEL_NAME="${MODEL_NAME:-Qwen3-14B}"
PORT="${PORT:-${SGLANG_PORT:-32000}}"
DIST_PORT="${DIST_PORT:-5320}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"
LOG_DIR="${LOG_DIR:-${WORK_HOME}/logs/service}"
MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.82}"
CONTEXT_LENGTH="${CONTEXT_LENGTH:-12000}"
CUDA_GRAPH_BS_MAX="${CUDA_GRAPH_BS_MAX:-128}"
SPEC_MODEL_PATH="${SPEC_MODEL_PATH:-/data/models/Qwen3-14B_eagle3}"
ENABLE_SPECULATIVE="${ENABLE_SPECULATIVE:-1}"
TOKENIZER_BACKEND="${TOKENIZER_BACKEND:-}"

SGLANG_ENABLE_SPECULATIVE="${SGLANG_ENABLE_SPECULATIVE:-${ENABLE_SPECULATIVE}}"
export SGLANG_ENABLE_SPEC_V2="${SGLANG_ENABLE_SPEC_V2:-1}"
mkdir -p "${LOG_DIR}" "${WORK_HOME}/traces"
TP_SIZE="${TP_SIZE:-1}"

MASTER_IP="$(hostname -I | awk '{print $1}')"
LOG_FILE="${LOG_DIR}/sglang_${MODEL_NAME}_$(date +%Y%m%d_%H%M%S).log"

SPEC_ARGS=()
if [[ "${SGLANG_ENABLE_SPECULATIVE}" == "1" || "${SGLANG_ENABLE_SPECULATIVE}" == "true" ]]; then
  export SGLANG_ENABLE_SPEC_V2=1
  SPEC_ARGS=(
    --speculative-draft-model-path "${SPEC_MODEL_PATH}"
    --speculative-algorithm EAGLE3
    --speculative-num-steps 3
    --speculative-eagle-topk 1
    --speculative-num-draft-tokens 4
  )
fi

TOKENIZER_ARGS=()
if [[ -n "${TOKENIZER_BACKEND}" ]]; then
  TOKENIZER_ARGS=(--tokenizer-backend "${TOKENIZER_BACKEND}")
fi

nohup "${SGLANG_PYTHON}" -m sglang.launch_server \
  --model-path "${MODEL_PATH}" \
  --served-model-name "${MODEL_NAME}" \
  --trust-remote-code \
  --tp-size "${TP_SIZE}" \
  --pp-size 1 \
  --host "${SGLANG_HOST}" \
  --port "${PORT}" \
  --dist-init-addr "${MASTER_IP}:${DIST_PORT}" \
  --attention-backend fa3 \
  --mm-attention-backend fa3 \
  --sampling-backend flashinfer \
  "${TOKENIZER_ARGS[@]}" \
  --mem-fraction-static "${MEM_FRACTION_STATIC}" \
  --cuda-graph-bs $(seq 1 "${CUDA_GRAPH_BS_MAX}") \
  --disable-cuda-graph-padding \
  --disable-piecewise-cuda-graph \
  --chunked-prefill-size -1 \
  --max-running-requests 128 \
  --context-length "${CONTEXT_LENGTH}" \
  --decode-log-interval 1 \
  "${SPEC_ARGS[@]}" \
  > "${LOG_FILE}" 2>&1 &

echo "SGLang ${MODEL_NAME} started, pid=$!, log=${LOG_FILE}"

Qwen3-32B

Qwen3-32B 使用单机4卡混合部署。

启动脚本 start_server.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export SGLANG_TORCH_PROFILER_DIR=/tmp/traces
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

SGLANG_PORT="${SGLANG_PORT:-31000}"
MASTER_IP=127.0.0.1
NODE_RANK=0
WORLD_SIZE=1
MODEL_PATH="${MODEL_PATH:-/data/models/Qwen3-32B}"
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="$WORK_HOME/output/$CURRENT_TIME/"
LOG_FILE="${LOG_DIR}/sglang_$(date '+%Y%m%d_%H%M%S').log"
LATEST_LOG_LINK="$WORK_HOME/output/latest"

mkdir -p "$LOG_DIR"
rm -f "$LATEST_LOG_LINK"
ln -s "$CURRENT_TIME/$(basename "$LOG_FILE")" "$LATEST_LOG_LINK"

nohup "$SGLANG_PYTHON" -m sglang.launch_server \
    --disable-piecewise-cuda-graph \
    --model "${MODEL_PATH}" \
    --served-model-name "${MODEL_NAME:-Qwen3-32B}" \
    --trust-remote-code \
    --cuda-graph-max-bs 256 \
    --disable-overlap-schedule \
    --tp-size 4 \
    --dp-size 1 \
    --mem-fraction-static 0.85 \
    --linear-attn-backend flashinfer \
    --tokenizer-backend fastokens \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:50122 \
    --nnodes 1 \
    --node-rank 0 \
    --chunked-prefill-size -1 \
    --port ${SGLANG_PORT} \
    --host "${SGLANG_HOST}" \
    --decode-log-interval 1 > "$LOG_FILE" 2>&1 &