Qwen3 Embedding & Reranker 模型部署

本章节包含 Qwen3 Embedding & Reranker 模型的部署脚本及说明。

• 参考 环境准备 进行环境检查、启动容器。

• 容器镜像地址参考 版本发布信息-镜像地址。

Qwen3 Embedding

以 Qwen3-Embedding-4B为例，模型权重下载。

启动脚本 start_server.sh

#!/bin/bash
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export PATH=/root/.virtualenvs/sglang-0.5.6/bin/:$PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1


# pkill -f "sgl.*"
# mthreads-gmi | grep -E '^[0-9]+\s+[0-9]+\s+' | awk '{print $2}' |
#sort -u | xargs kill -9
# sleep 3

get_master_ip() {
    local hostname_ip=$(hostname -I 2>/dev/null | awk '{print $1}')
    
    if [ -n "$hostname_ip" ] && [ "$hostname_ip" != "127.0.0.1" ]; then
        echo "$hostname_ip"
        return
    fi
    
}

CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
SGLANG_DECODE_PORT=22230
MASTER_IP=$(get_master_ip)
NODE_RANK=0
WORLD_SIZE=1

WORK_HOME="$PWD"
LOG_DIR=$WORK_HOME/output/$CURRENT_TIME/
mkdir -p "$LOG_DIR"

export MUSA_VISIBLE_DEVICES=4
export SGLANG_TORCH_PROFILER_DIR=$WORK_HOME/traces
MODEL_PATH=/models/Qwen3-Embedding-4B

python3 -m sglang.launch_server \
            --model-path $MODEL_PATH \
            --is-embedding \
            --disable-piecewise-cuda-graph \
            --served-model-name base-model \
            --mem-fraction-static 0.76 \
            --host 0.0.0.0 \
            --port 23230 \
            --attention-backend fa3 \
            --moe-runner-backend deep_gemm \
            --tp-size 1 \
            --max-running-requests 32 \
            --sampling-backend flashinfer \
            --disable-radix-cache \
            --json-model-override-args '{"matryoshka_dimensions": [128, 256, 512, 1024, 1536]}' \
            --chunked-prefill-size -1 > "${LOG_DIR}/${MASTER_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

echo -e "Main log file: \033[34m$LOG_DIR\033[0m"

验证程序 curl.py

import openai
client = openai.Client(
    base_url="http://127.0.0.1:23230/v1", 
    api_key="EMPTY"  # SGlang不校验key，填任意字符串均可
)
 
# # 单条文本嵌入
# response = client.embeddings.create(
#     model="Qwen3-Embedding-4B",
#     input="今天天气真好，适合写代码"
# )
# print("向量维度:", len(response.data[0].embedding))
# print("前5维数值:", response.data[0].embedding[:5])

response = client.embeddings.create(
    model="Qwen3-reranker-4B",
    input=[
        "苹果手机的iOS系统很流畅",
        "华为手机搭载鸿蒙OS，生态协同强",
        "Python的pandas库适合数据清洗",
        "Transformer架构是大模型的基础"
    ],
    dimensions=1024  # 关键！指定输出维度
)
 
for i, item in enumerate(response.data):
    print(f"句子{i+1}向量维度:", len(item.embedding))

print("向量维度:", len(response.data[0].embedding))
print("前5维数值:", response.data[0].embedding[:5])

Qwen3-Reranker

以 Qwen3-Reranker-0.6B为例，模型权重下载。

启动脚本 start_server.sh

#!/bin/bash
export PATH=/root/.virtualenvs/sglang-default/bin/:$PATH
export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/:/usr/local/musa/lib:$LD_LIBRARY_PATH
export MUSA_LAUNCH_BLOCKING=0
export MUSA_ENABLE_LLC_OPT=1
export MCCL_IB_GID_INDEX=3
export MCCL_NET_SHARED_BUFFERS=0
export MCCL_PROTOS=2
export GLOO_SOCKET_IFNAME=bond0
export TP_SOCKET_IFNAME=bond0
export VLLM_PATCH_MUSA_CUSTOM_OPS=1


# pkill -f "sgl.*"
# mthreads-gmi | grep -E '^[0-9]+\s+[0-9]+\s+' | awk '{print $2}' |
#sort -u | xargs kill -9
# sleep 3

get_master_ip() {
    local hostname_ip=$(hostname -I 2>/dev/null | awk '{print $1}')
    
    if [ -n "$hostname_ip" ] && [ "$hostname_ip" != "127.0.0.1" ]; then
        echo "$hostname_ip"
        return
    fi
    
}

CURRENT_TIME=$(date "+%Y%m%d_%H%M%S")
SGLANG_DECODE_PORT=24440
#MASTER_IP=127.0.0.1
MASTER_IP=$(get_master_ip)
NODE_RANK=0
WORLD_SIZE=1
WORK_HOME="$PWD"
LOG_DIR=$WORK_HOME/output/$CURRENT_TIME/
mkdir -p "$LOG_DIR"

export MUSA_VISIBLE_DEVICES=6
export SGLANG_TORCH_PROFILER_DIR=$WORK_HOME/traces
MODEL_PATH=/data/models/Qwen3-Reranker-0.6B/
nohup python3 -m sglang.launch_server \
      --model-path ${MODEL_PATH} \
      --trust-remote-code \
      --disable-overlap-schedule \
      --disable-cuda-graph \
      --disable-piecewise-cuda-graph \
      --tp-size 1 \
      --mem-fraction-static 0.8 \
      --attention-backend fa3 \
      --mm-attention-backend fa3 \
      --moe-runner-backend deep_gemm \
      --sampling-backend flashinfer \
      --dist-init-addr ${MASTER_IP}:56666 \
      --nnodes ${WORLD_SIZE} \
      --node-rank ${NODE_RANK} \
      --disable-radix-cache \
      --port ${SGLANG_DECODE_PORT} \
      --host 0.0.0.0 \
      --decode-log-interval 1 > "${LOG_DIR}/${MASTER_IP}_$(date '+%Y%m%d_%H%M%S').log" 2>&1 &

echo -e "Main log file: \033[34m$LOG_DIR\033[0m"

验证命令

## 单条curl
curl -X POST http://127.0.0.1:24440/v1/rerank \
-H "Content-Type: application/json"  \
-d '{ "query": "法国首都是哪里？", "documents": [ "法国的首都是巴黎。", "德国的首都是柏林。", "香蕉是黄色的水果。" ], "return_documents": true }'

curl http://127.0.0.1:24440/v1/score \
-H "Content-Type: application/json"  \
-d '{ "model": "Qwen/Qwen3-Reranker-0.6B", "query": "What is 2+2?", "items": ["4", "5"], "label_token_ids": [10967, 2152] }'