跳到主要内容

Qwen3 Dense 模型部署

本章节包含 Qwen3 Dense 系列模型的部署脚本及说明。支持 BF16 和 FP8 两种权重，部署时注意替换权重文件。

Qwen3-8B​

投机采样使用了 Draft Model，模型链接（Qwen3-8B_eagle3）

启动脚本 start_server.sh

#!/usr/bin/env bash
set -euo pipefail

SGLANG_VENV="${SGLANG_VENV:-/root/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
if [[ -f "${SGLANG_VENV}/bin/activate" ]]; then
  # shellcheck disable=SC1091
  source "${SGLANG_VENV}/bin/activate"
fi

export PATH="${SGLANG_VENV}/bin:${PATH}"
export LD_LIBRARY_PATH="/usr/local/musa/lib:/usr/local/openmpi/lib:/usr/local/musa/mudnn/lib:/usr/local/mtshmem/lib:${LD_LIBRARY_PATH:-}"
export MUSA_VISIBLE_DEVICES="${MUSA_VISIBLE_DEVICES:-0}"
export MTHREADS_VISIBLE_DEVICES="${MTHREADS_VISIBLE_DEVICES:-0}"
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_USE_DEEPGEMM_BMM=0
export SGLANG_MOE_PADDING=1
export SGLANG_USE_MUSA_FUSED_KERNEL=1

WORK_HOME="${WORK_HOME:-$PWD}"
MODEL_PATH="${MODEL_PATH:-/data/models/Qwen3-8B-FP8}"
MODEL_NAME="${MODEL_NAME:-Qwen3-8B-FP8}"
PORT="${PORT:-${SGLANG_PORT:-42251}}"
DIST_PORT="${DIST_PORT:-1264}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"
LOG_DIR="${LOG_DIR:-${WORK_HOME}/logs/service}"
SPEC_MODEL_PATH="${SPEC_MODEL_PATH:-/data/models/Qwen3-8B_eagle3}"
ENABLE_SPECULATIVE="${ENABLE_SPECULATIVE:-1}"
TOKENIZER_BACKEND="${TOKENIZER_BACKEND:-fastokens}"

SGLANG_ENABLE_SPECULATIVE="${SGLANG_ENABLE_SPECULATIVE:-${ENABLE_SPECULATIVE}}"
export SGLANG_ENABLE_SPEC_V2="${SGLANG_ENABLE_SPEC_V2:-0}"
mkdir -p "${LOG_DIR}" "${WORK_HOME}/traces"
TP_SIZE="${TP_SIZE:-1}"

MASTER_IP="$(hostname -I | awk '{print $1}')"
LOG_FILE="${LOG_DIR}/qwen3_8b_2500_1500_short_$(date +%Y%m%d_%H%M%S).log"

SPEC_ARGS=()
if [[ "${SGLANG_ENABLE_SPECULATIVE}" == "1" || "${SGLANG_ENABLE_SPECULATIVE}" == "true" ]]; then
  export SGLANG_ENABLE_SPEC_V2=1
  SPEC_ARGS=(
    --speculative-draft-model-path "${SPEC_MODEL_PATH}"
    --speculative-algorithm EAGLE3
    --speculative-num-steps 3
    --speculative-eagle-topk 1
    --speculative-num-draft-tokens 4
  )
fi

TOKENIZER_ARGS=()
if [[ -n "${TOKENIZER_BACKEND}" ]]; then
  TOKENIZER_ARGS=(--tokenizer-backend "${TOKENIZER_BACKEND}")
fi

nohup "${SGLANG_PYTHON}" -m sglang.launch_server \
  --model-path "${MODEL_PATH}" \
  --served-model-name "${MODEL_NAME}" \
  --trust-remote-code \
  --tp-size "${TP_SIZE}" \
  --pp-size 1 \
  --host "${SGLANG_HOST}" \
  --port "${PORT}" \
  --dist-init-addr "${MASTER_IP}:${DIST_PORT}" \
  --attention-backend fa3 \
  --mm-attention-backend fa3 \
  --tokenizer-backend fastokens \
  --linear-attn-backend flashinfer \
  --sampling-backend flashinfer \
  "${TOKENIZER_ARGS[@]}" \
  --mem-fraction-static 0.7 \
  --cuda-graph-max-bs 256 \
  --disable-piecewise-cuda-graph \
  --chunked-prefill-size -1 \
  --max-running-requests 256 \
  --decode-log-interval 1 \
  "${SPEC_ARGS[@]}" \
  --reasoning-parser qwen3 \
  --tool-call-parser qwen25 \
  > "${LOG_FILE}" 2>&1 &

echo "SGLang Qwen3-8B started, pid=$!, log=${LOG_FILE}"

Qwen3-14B​

启动脚本 start_server.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export SGLANG_ENABLE_SPEC_V2=1
MODEL_PATH="${MODEL_PATH:-/data/models/Qwen3-14B-FP8}"
SGLANG_PORT="${SGLANG_PORT:-31000}"
TP_SIZE="${TP_SIZE:-1}"
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="$WORK_HOME/output/$CURRENT_TIME/"
LOG_FILE="${LOG_DIR}/sglang_$(date '+%Y%m%d_%H%M%S').log"
LATEST_LOG_LINK="$WORK_HOME/output/latest"

mkdir -p "$LOG_DIR"
rm -f "$LATEST_LOG_LINK"
ln -s "$CURRENT_TIME/$(basename "$LOG_FILE")" "$LATEST_LOG_LINK"

nohup "$SGLANG_PYTHON" -m sglang.launch_server \
  --model-path "$MODEL_PATH" \
  --served-model-name base-model \
  --mem-fraction-static 0.75 \
  --cuda-graph-bs $(seq 1 64) \
  --disable-piecewise-cuda-graph \
  --disable-overlap-schedule \
  --linear-attn-backend flashinfer \
  --tokenizer-backend fastokens \
  --host "${SGLANG_HOST}" \
  --port ${SGLANG_PORT} \
  --attention-backend fa3 \
  --tp-size "$TP_SIZE" \
  --max-running-requests 256 \
  --sampling-backend flashinfer \
  --chunked-prefill-size 8192 \
  --reasoning-parser qwen3 \
  --tool-call-parser qwen25 \
  > "$LOG_FILE" 2>&1 &

Qwen3-32B​

Qwen3-32B 可以使用单机4卡混合部署，也可以在1台机器内做PD分离部署。

四卡 tp4 混合部署​

如果使用投机采样 Draft Model，模型链接（Qwen3-32B_eagle3）

启动脚本 start_server.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export SGLANG_TORCH_PROFILER_DIR=/tmp/traces
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
export LD_LIBRARY_PATH=/usr/local/musa/lib:${LD_LIBRARY_PATH}

SGLANG_PORT="${SGLANG_PORT:-31000}"
MASTER_IP=127.0.0.1
NODE_RANK=0
WORLD_SIZE=1
MODEL_PATH=/data/models/Qwen3-32B/
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="$WORK_HOME/output/$CURRENT_TIME/"
LOG_FILE="${LOG_DIR}/sglang_$(date '+%Y%m%d_%H%M%S').log"
LATEST_LOG_LINK="$WORK_HOME/output/latest"

mkdir -p "$LOG_DIR"
rm -f "$LATEST_LOG_LINK"
ln -s "$CURRENT_TIME/$(basename "$LOG_FILE")" "$LATEST_LOG_LINK"

nohup "$SGLANG_PYTHON" -m sglang.launch_server \
    --model ${MODEL_PATH} \
    --trust-remote-code \
    --cuda-graph-max-bs 256 \
    --disable-piecewise-cuda-graph \
    --disable-overlap-schedule \
    --tp-size 4 \
    --dp-size 1 \
    --mem-fraction-static 0.8 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --tokenizer-backend fastokens \
    --linear-attn-backend flashinfer \
    --sampling-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:50122 \
    --nnodes 1 \
    --node-rank 0 \
    --chunked-prefill-size -1 \
    --port ${SGLANG_PORT} \
    --host "${SGLANG_HOST}" \
    --decode-log-interval 1 \
    --reasoning-parser qwen3 \
    --tool-call-parser qwen25 \
    > "$LOG_FILE" 2>&1 &

单机PD分离部署​

该示例使用1台机器，4张卡做Prefill，4张卡做Decode。

Prefill启动脚本 prefill_server.sh

export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH

export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1

export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0

export MC_ENABLE_DEST_DEVICE_AFFINITY=1

export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_DEEP_GEMM_BLOCK_M=256
# 如果模型为 BF16 数据类型则打开该环境变量
export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_ENABLE_TORCH_INFERENCE_MODE=true

export SGLANG_DISAGGREGATION_QUEUE_SIZE=8
export SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=16
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600

export SGLANG_TORCH_PROFILER_DIR=/tmp/traces
export VLLM_PATCH_MUSA_CUSTOM_OPS=1

export SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True
export MC_FORCE_MNNVL=True

mthreads-gmi | grep -E '^[0-9]+\s+[0-9]+\s+' | awk '{print $2}' | sort -u | xargs kill -9 &> /dev/null
sleep 3


SGLANG_PREFILL_PORT=32011
SGLANG_DECODE_PORT=32022
SGLANG_ROUTER_PORT=30000


MASTER_IP=127.0.0.1
NODE_RANK=0
WORLD_SIZE=1
MODEL_PATH=/data/models/Qwen3-32B

log_path=logs-sglang-server/$(hostname)
mkdir -p $log_path
times_now=$(date "+%Y%m%d_%H%M")

prefill_log_file=${times_now}_prefill.log
decode_log_file=${times_now}_decode.log
router_log_file=${times_now}_router.log


export MUSA_VISIBLE_DEVICES=0,1,2,3
nohup python3 -m sglang.launch_server \
    --model ${MODEL_PATH} \
    --trust-remote-code \
    --disable-cuda-graph \
    --disable-overlap-schedule \
    --tp-size 4 \
    --dp-size 1 \
    --mem-fraction-static 0.85 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --dist-init-addr ${MASTER_IP}:50011 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size 8192 \
    --enable-cache-report \
    --schedule-conservativeness 1 \
    --max-running-requests 64 \
    --max-prefill-tokens 4096 \
    --port ${SGLANG_PREFILL_PORT} \
    --host 0.0.0.0 \
    --load-balance-method round_robin \
    --disaggregation-mode prefill \
    --disaggregation-ib-device mlx5_bond_2 \
    --reasoning-parser qwen3 \
    --tool-call-parser qwen25 \
    > ${log_path}/${prefill_log_file} 2>&1 &
sleep 10

Decode启动脚本 decode_server.sh

export SGLANG_DEEPEP_LL_USE_NVLINK=1
export SGLANG_DEEPEP_LL_DISABLE_RECV_HOOK=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128

export MUSA_VISIBLE_DEVICES=4,5,6,7

nohup python3 -m sglang.launch_server \
    --model ${MODEL_PATH} \
    --trust-remote-code \
    --cuda-graph-max-bs 256  \
    --tp-size 4 \
    --dp-size 1 \
    --mem-fraction-static 0.8 \
    --attention-backend fa3 \
    --mm-attention-backend fa3 \
    --sampling-backend flashinfer \
    --max-running-requests 256 \
    --dist-init-addr ${MASTER_IP}:50022 \
    --nnodes ${WORLD_SIZE} \
    --node-rank ${NODE_RANK} \
    --chunked-prefill-size -1 \
    --port ${SGLANG_DECODE_PORT} \
    --host 0.0.0.0 \
    --decode-log-interval 1 \
    --load-balance-method round_robin \
    --prefill-round-robin-balance \
    --disaggregation-mode decode \
    --speculative-algorithm EAGLE3 \
    --speculative-num-steps 1 \
    --speculative-eagle-topk 1 \
    --speculative-num-draft-tokens 2 \
    --speculative-draft-model-path /data/models/Qwen3-32B_eagle3/ \
    --disaggregation-ib-device mlx5_bond_2 \
    --reasoning-parser qwen3 \
    --tool-call-parser qwen25 \
    > ${log_path}/${decode_log_file} 2>&1 &



kill -9 $(lsof -t -i:${SGLANG_ROUTER_PORT})


PREFILL_IP="http://127.0.0.1:$SGLANG_PREFILL_PORT"
DECODE_IP="http://127.0.0.1:$SGLANG_DECODE_PORT"

nohup python -m sglang_router.launch_router \
    --pd-disaggregation \
    --mini-lb \
    --prefill $PREFILL_IP \
    --decode $DECODE_IP \
    --host 0.0.0.0 \
    --request-timeout-secs 7200 \
    --port $SGLANG_ROUTER_PORT > ${log_path}/${router_log_file} 2>&1 &

while [ 0 -eq 0 ];do
	grep "The server is fired up and ready to roll!" ${log_path}/${decode_log_file} > /dev/null 2>&1
	if [ $? -eq 0 ];then
		break
	fi
done
echo "All logs are stored in ./logs"

版权所有 2024 摩尔线程，本文档受国际版权法保护。

摩尔线程和摩尔线程徽标是摩尔线程智能科技 (北京)有限责任公司的注册商标。

免责声明

本文档提供有关摩尔线程产品的信息。本文档并未授权任何知识产权的许可，并未以明示或暗示，或以禁止反言或其他方式授予任何知识产权许可。