Qwen3.6 模型部署

本章节包含 Qwen3.6 系列模型的部署脚本及说明。

模型权重下载。
参考环境准备进行环境检查、启动容器。
容器镜像地址参考版本发布信息-镜像地址。

Qwen3.6-27B

2卡部署（tp=2）示例如下

启动脚本 start_server.sh

#!/usr/bin/env bash
set -euo pipefail

MODEL_NAME="${MODEL_NAME:-Qwen3.6-27B}"
MODEL_PATH="${MODEL_PATH:-/data/models/Qwen3.6-27B}"
PORT="${PORT:-32106}"
DIST_INIT_ADDR="${DIST_INIT_ADDR:-10.20.35.95:6716}"
MUSA_VISIBLE_DEVICES="${MUSA_VISIBLE_DEVICES:-2,3}"
MTHREADS_VISIBLE_DEVICES="${MTHREADS_VISIBLE_DEVICES:-${MUSA_VISIBLE_DEVICES}}"
PYTHON_BIN="${PYTHON_BIN:-/root/.virtualenvs/sglang-default/bin/python3}"
LOG_DIR="${LOG_DIR:-/data/workspace/logs}"

mkdir -p "${LOG_DIR}"
source /root/.virtualenvs/sglang-default/bin/activate

export MUSA_VISIBLE_DEVICES
export MTHREADS_VISIBLE_DEVICES
export PATH="/root/.virtualenvs/sglang-default/bin:${PATH}"
export LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu:/usr/local/musa/lib:/usr/local/musa/lib64:/usr/local/openmpi/lib:/usr/local/musa/mudnn/lib:/usr/local/mtshmem/lib:${LD_LIBRARY_PATH:-}"
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_USE_DEEPGEMM_BMM=0
export SGLANG_MOE_PADDING=1
export SGLANG_USE_MUSA_FUSED_KERNEL=1
export TOKENIZERS_PARALLELISM=false
export OPENAI_API_KEY="${OPENAI_API_KEY:-EMPTY}"

"${PYTHON_BIN}" -m pip show fastokens >/dev/null 2>&1 || \
  "${PYTHON_BIN}" -m pip install 'fastokens>=0.1.1,<0.2.0'

nohup "${PYTHON_BIN}" -m sglang.launch_server \
  --model-path "${MODEL_PATH}" \
  --served-model-name "${MODEL_NAME}" \
  --trust-remote-code \
  --tp-size 2 \
  --host 0.0.0.0 \
  --port "${PORT}" \
  --attention-backend fa3 \
  --mm-attention-backend fa3 \
  --sampling-backend flashinfer \
  --linear-attn-backend flashinfer \
  --tokenizer-backend fastokens \
  --mem-fraction-static 0.80 \
  --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 \
    17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 \
    33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 \
    49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 \
  --disable-cuda-graph-padding \
  --disable-radix-cache \
  --mamba-scheduler-strategy no_buffer \
  --chunked-prefill-size -1 \
  --disable-overlap-schedule \
  --watchdog-timeout 1200 \
  --soft-watchdog-timeout 1200 \
  --decode-log-interval 1 \
  --reasoning-parser qwen3 \
  --tool-call-parser qwen3_coder \
  > "${LOG_DIR}/service.log"  2>&1 &

Qwen3.6-35B-A3B

单卡部署（tp=1）示例如下

启动脚本 start_server.sh

#!/usr/bin/env bash
set -euo pipefail
source /root/.virtualenvs/sglang-default/bin/activate
export PATH=/root/.virtualenvs/sglang-default/bin:${PATH}
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:/usr/local/musa/lib:/usr/local/musa/lib64:/usr/local/openmpi/lib:/usr/local/musa/mudnn/lib:/usr/local/mtshmem/lib:${LD_LIBRARY_PATH:-}
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_USE_DEEPGEMM_BMM=0
export SGLANG_MOE_PADDING=1
export SGLANG_USE_MUSA_FUSED_KERNEL=1
export TOKENIZERS_PARALLELISM=false
export OPENAI_API_KEY=EMPTY

MODEL_PATH=${MODEL_PATH:-/data/models/Qwen3.6-35B-A3B}
PORT=${PORT:-32001}
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
WORK_HOME="$PWD"
LOG_DIR="${LOG_DIR:-$WORK_HOME/output/$CURRENT_TIME}"
LOG_FILE="${LOG_FILE:-$LOG_DIR/sglang_${CURRENT_TIME}.log}"
mkdir -p "$LOG_DIR"

nohup /root/.virtualenvs/sglang-default/bin/python3 -m sglang.launch_server \
  --model-path "${MODEL_PATH}" \
  --served-model-name Qwen3.6-35B-A3B \
  --trust-remote-code \
  --tp-size 1   \
  --ep-size 1   \
  --host 0.0.0.0   \
  --port "${PORT}"   \
  --attention-backend fa3 \
  --mm-attention-backend fa3  \
  --moe-runner-backend auto  \
  --sampling-backend flashinfer  \
  --linear-attn-backend flashinfer  \
  --tokenizer-backend fastokens  \
  --mem-fraction-static 0.9  \
  --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64  \
  --disable-cuda-graph-padding  \
  --disable-radix-cache  \
  --mamba-scheduler-strategy no_buffer  \
  --chunked-prefill-size -1  \
  --disable-overlap-schedule  \
  --watchdog-timeout 1200  \
  --soft-watchdog-timeout 1200  \
  --decode-log-interval 1 > "${LOG_FILE}" 2>&1 &

4卡部署（tp=4）示例如下:

启动脚本 start_server.sh

#!/usr/bin/env bash
set -euo pipefail

MODEL_NAME="${MODEL_NAME:-Qwen3.6-35B-A3B}"
MODEL_PATH="${MODEL_PATH:-/data/models/Qwen3.6-35B-A3B}"
PORT="${PORT:-32001}"
DIST_INIT_ADDR="${DIST_INIT_ADDR:-$(hostname -I 2>/dev/null | awk '{print $1}'):6601}"
MUSA_VISIBLE_DEVICES="${MUSA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
MTHREADS_VISIBLE_DEVICES="${MTHREADS_VISIBLE_DEVICES:-${MUSA_VISIBLE_DEVICES}}"
PYTHON_BIN="${PYTHON_BIN:-/root/.virtualenvs/sglang-default/bin/python3}"
# LOG_DIR="${LOG_DIR:-/mnt/data/fanxy/Qwen3.6-35B-A3B}"
WORK_HOME="$PWD"
CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
LOG_DIR=$WORK_HOME/output/$CURRENT_TIME/

mkdir -p "${LOG_DIR}"
source /root/.virtualenvs/sglang-default/bin/activate

export MUSA_VISIBLE_DEVICES
export MTHREADS_VISIBLE_DEVICES
export PATH="/root/.virtualenvs/sglang-default/bin:${PATH}"
export LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu:/usr/local/musa/lib:/usr/local/musa/lib64:/usr/local/openmpi/lib:/usr/local/musa/mudnn/lib:/usr/local/mtshmem/lib:${LD_LIBRARY_PATH:-}"
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export VLLM_PATCH_MUSA_CUSTOM_OPS=1
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_USE_DEEPGEMM_BMM=0
export SGLANG_MOE_PADDING=1
export SGLANG_USE_MUSA_FUSED_KERNEL=1
export TOKENIZERS_PARALLELISM=false
export OPENAI_API_KEY="${OPENAI_API_KEY:-EMPTY}"
export SGLANG_ENABLE_SPEC_V2=1


"${PYTHON_BIN}" -m pip show fastokens >/dev/null 2>&1 || \
  "${PYTHON_BIN}" -m pip install 'fastokens>=0.1.1,<0.2.0'

exec "${PYTHON_BIN}" -m sglang.launch_server \
  --model-path "${MODEL_PATH}" \
  --served-model-name "${MODEL_NAME}" \
  --host 0.0.0.0 \
  --port 30000 \
  --trust-remote-code \
  --tp-size 4 \
  --attention-backend fa3 \
  --mm-attention-backend fa3 \
  --linear-attn-backend flashinfer \
  --chunked-prefill-size 16384 \
  --moe-runner-backend auto \
  --cuda-graph-max-bs 128 \
  --mem-fraction-static 0.78 \
  --sampling-backend flashinfer \
  --tokenizer-backend fastokens \
  --decode-log-interval 1 \
  --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64  \
  --disable-cuda-graph-padding  \
  --disable-radix-cache  \
  --disable-overlap-schedule  \
  --watchdog-timeout 1200  \
  --mamba-scheduler-strategy extra_buffer \
  --speculative-algo NEXTN \
  --speculative-num-steps 2 \
  --speculative-eagle-topk 1 \
  --speculative-num-draft-tokens 3 \
  --reasoning-parser qwen3 \
  --tool-call-parser qwen3_coder  \
  2>&1 | tee "${LOG_DIR}/service.log"

Qwen3.6-27B​

Qwen3.6-35B-A3B​

这篇文章对您有帮助吗？

Qwen3.6-27B

Qwen3.6-35B-A3B