Qwen3.6 模型部署

本章节包含 Qwen3.6 系列模型的部署脚本及说明。

• 模型权重下载。

• 参考 环境准备 进行环境检查、启动容器。

• 容器镜像地址参考 版本发布信息-镜像地址。

Qwen3.6-27B

2卡部署 （tp=2） 示例如下。脚本未显式设置 --served-model-name，服务名默认为模型路径的目录名（Qwen3.6-27B）；如需自定义，在启动命令中补充 --served-model-name <名称>，并保证后续请求中的 model 字段与之完全一致。

启动脚本 start_server.sh

#!/bin/bash
SGLANG_VENV="${SGLANG_VENV:-$HOME/.virtualenvs/sglang-default}"
SGLANG_PYTHON="${SGLANG_PYTHON:-${SGLANG_VENV}/bin/python3}"
SGLANG_HOST="${SGLANG_HOST:-0.0.0.0}"
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
SGLANG_ENABLE_SPECULATIVE="${SGLANG_ENABLE_SPECULATIVE:-1}"

# 清理缓存
rm -rf ~/.triton/cache
rm -rf /tmp/*
# pkill -f "sgl.*"
# mthreads-gmi | grep -E '^[0-9]+\s+[0-9]+\s+' | awk '{print $2}' |
#sort -u | xargs kill -9
# sleep 3

get_master_ip() {
    local hostname_ip=$(hostname -I 2>/dev/null | awk '{print $1}')

    if [ -n "$hostname_ip" ] && [ "$hostname_ip" != "127.0.0.1" ]; then
        echo "$hostname_ip"
        return
    fi

}

CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
SGLANG_PORT="${SGLANG_PORT:-31000}"
#MASTER_IP=127.0.0.1
MASTER_IP=$(get_master_ip)
NODE_RANK=0
WORLD_SIZE=1
WORK_HOME="$PWD"
LOG_DIR=$WORK_HOME/output/$CURRENT_TIME/
mkdir -p "$LOG_DIR"

export SGLANG_TORCH_PROFILER_DIR=$WORK_HOME/traces
MODEL_PATH="${MODEL_PATH:-/data/models/Qwen3.6-27B}"
cmd=(
      "$SGLANG_PYTHON" -m sglang.launch_server
      --model-path "${MODEL_PATH}"
      --trust-remote-code
      --cuda-graph-max-bs 192
      --disable-cuda-graph-padding
      --disable-piecewise-cuda-graph
      --disable-overlap-schedule
      --tp-size 2
      --pp-size 1
      --mem-fraction-static 0.78
      --attention-backend fa3
      --sampling-backend flashinfer   
      --linear-attn-backend flashinfer  
      --tokenizer-backend fastokens 
      --enable-cache-report
      --mm-attention-backend fa3
      --moe-runner-backend auto
      --dist-init-addr ${MASTER_IP}:50022
      --nnodes ${WORLD_SIZE}
      --node-rank ${NODE_RANK}
      --chunked-prefill-size 4096
      --max-running-requests 192
      --max-prefill-tokens 8192
      --port ${SGLANG_PORT}
      --host "${SGLANG_HOST}"
      --decode-log-interval 1
)

if [[ "$SGLANG_ENABLE_SPECULATIVE" == "1" ]]; then
      cmd+=(
      --speculative-algorithm NEXTN
      --speculative-num-steps 1
      --speculative-eagle-topk 1
      --speculative-num-draft-tokens 2
      )
fi

nohup "${cmd[@]}" > "${LOG_DIR}/${MASTER_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

echo -e "Main log file: \033[34m$LOG_DIR\033[0m"

Qwen3.6-35B-A3B

单卡部署 （tp=1） 示例如下

启动脚本 start_server.sh

#!/usr/bin/env bash
set -euo pipefail
source /root/.virtualenvs/sglang-default/bin/activate
export PATH=/root/.virtualenvs/sglang-default/bin:${PATH}
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:/usr/local/musa/lib:/usr/local/musa/lib64:/usr/local/openmpi/lib:/usr/local/musa/mudnn/lib:/usr/local/mtshmem/lib:${LD_LIBRARY_PATH:-}
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_USE_DEEPGEMM_BMM=0
export SGLANG_MOE_PADDING=1
export SGLANG_USE_MUSA_FUSED_KERNEL=1
export TOKENIZERS_PARALLELISM=false
export OPENAI_API_KEY=EMPTY

MODEL_PATH=${MODEL_PATH:-/data/models/Qwen3.6-35B-A3B}
PORT=${PORT:-32001}
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="${LOG_DIR:-$WORK_HOME/output/$CURRENT_TIME}"
LOG_FILE="${LOG_FILE:-$LOG_DIR/sglang_${CURRENT_TIME}.log}"
mkdir -p "$LOG_DIR"

nohup /root/.virtualenvs/sglang-default/bin/python3 -m sglang.launch_server \
  --model-path "${MODEL_PATH}" \
  --served-model-name Qwen3.6-35B-A3B \
  --trust-remote-code \
  --tp-size 1   \
  --ep-size 1   \
  --host 0.0.0.0   \
  --port "${PORT}"   \
  --attention-backend fa3 \
  --mm-attention-backend fa3  \
  --moe-runner-backend auto  \
  --sampling-backend flashinfer  \
  --linear-attn-backend flashinfer  \
  --tokenizer-backend fastokens  \
  --mem-fraction-static 0.9  \
  --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64  \
  --mamba-scheduler-strategy extra_buffer \
  --chunked-prefill-size -1  \
  --disable-overlap-schedule  \
  --watchdog-timeout 1200  \
  --soft-watchdog-timeout 1200  \
  --decode-log-interval 1 > "${LOG_FILE}" 2>&1 &

4卡部署 （tp=4） 示例如下:

启动脚本 start_server.sh

#!/usr/bin/env bash
set -euo pipefail

MODEL_NAME="${MODEL_NAME:-Qwen3.6-35B-A3B}"
MODEL_PATH="${MODEL_PATH:-/data/models/Qwen3.6-35B-A3B}"
PORT="${PORT:-32001}"
DIST_INIT_ADDR="${DIST_INIT_ADDR:-$(hostname -I 2>/dev/null | awk '{print $1}'):6601}"
MUSA_VISIBLE_DEVICES="${MUSA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
MTHREADS_VISIBLE_DEVICES="${MTHREADS_VISIBLE_DEVICES:-${MUSA_VISIBLE_DEVICES}}"
PYTHON_BIN="${PYTHON_BIN:-/root/.virtualenvs/sglang-default/bin/python3}"
# LOG_DIR="${LOG_DIR:-/data/workspace/logs/qwen3.6-35b-a3b}"
WORK_HOME="$PWD"
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
LOG_DIR=$WORK_HOME/output/$CURRENT_TIME/

mkdir -p "${LOG_DIR}"
source /root/.virtualenvs/sglang-default/bin/activate

export MUSA_VISIBLE_DEVICES
export MTHREADS_VISIBLE_DEVICES
export PATH="/root/.virtualenvs/sglang-default/bin:${PATH}"
export LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu:/usr/local/musa/lib:/usr/local/musa/lib64:/usr/local/openmpi/lib:/usr/local/musa/mudnn/lib:/usr/local/mtshmem/lib:${LD_LIBRARY_PATH:-}"
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_USE_DEEPGEMM_BMM=0
export SGLANG_MOE_PADDING=1
export SGLANG_USE_MUSA_FUSED_KERNEL=1
export TOKENIZERS_PARALLELISM=false
export OPENAI_API_KEY="${OPENAI_API_KEY:-EMPTY}"
export SGLANG_ENABLE_SPEC_V2=1


"${PYTHON_BIN}" -m pip show fastokens >/dev/null 2>&1 || \
  "${PYTHON_BIN}" -m pip install 'fastokens>=0.1.1,<0.2.0'

exec "${PYTHON_BIN}" -m sglang.launch_server \
  --model-path "${MODEL_PATH}" \
  --served-model-name "${MODEL_NAME}" \
  --host 0.0.0.0 \
  --port 30000 \
  --trust-remote-code \
  --tp-size 4 \
  --attention-backend fa3 \
  --mm-attention-backend fa3 \
  --linear-attn-backend flashinfer \
  --chunked-prefill-size 16384 \
  --moe-runner-backend auto \
  --cuda-graph-max-bs 128 \
  --mem-fraction-static 0.78 \
  --sampling-backend flashinfer \
  --tokenizer-backend fastokens \
  --decode-log-interval 1 \
  --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64  \
  --mamba-scheduler-strategy extra_buffer \
  --disable-overlap-schedule  \
  --watchdog-timeout 1200  \
  --speculative-algorithm NEXTN \
  --speculative-num-steps 2 \
  --speculative-eagle-topk 1 \
  --speculative-num-draft-tokens 3 \
  --reasoning-parser qwen3 \
  --tool-call-parser qwen3_coder  \
  2>&1 | tee "${LOG_DIR}/service.log"