1.系统环境
- NVIDIA T4 * 2 /16G * 2 Driver Version: 535.154.05 CUDA Version: 12.2
2.打开modelscope选择对应的模型，我选择Qwen3.5-4B
https://www.modelscope.cn/models
3.vllm镜像下载，使用vllm容器加载模型
docker pull vllm/vllm-openai:latest
4.使用vllm镜像下载Qwen3.5-4B模型文件
docker run –rm -it \\
–gpus all \\
–entrypoint /bin/bash \\
–pids-limit -1 \\
–security-opt seccomp=unconfined \\
-v /root/lipengcheng/qwen35_4b:/models \\
-e OMP_NUM_THREADS=8 \\
vllm/vllm-openai:latest \\
-c "pip install modelscope && python3 -c \\"from modelscope import snapshot_download; snapshot_download('Qwen/Qwen3.5-4B', cache_dir='/models')\\""
5.使用vllm加载Qwen3.5-4B模型
version: '3.8'
services:
vllm-qwen:
image: vllm/vllm-openai:latest
container_name: vllm-qwen35-4b
privileged: true
deploy:
resources:
reservations:
devices:
– driver: nvidia
count: 2
capabilities: [gpu]
volumes:
– /root/lipengcheng/qwen35_4b/Qwen:/models
ports:
– "23333:8000"
environment:
– NVIDIA_VISIBLE_DEVICES=all
command: >
–model /models/Qwen3___5-4B
–host 0.0.0.0
–port 8000
–dtype half
–served-model-name qwen3.5-4b
–max-model-len 8192
–tensor-parallel-size 2
–default-chat-template-kwargs '{"enable_thinking": false}'
restart: unless-stopped
6.测试
curl http://localhost:23333/v1/chat/completions \\
-H "Content-Type: application/json" \\
-d '{
"model": "qwen3.5-4b",
"messages": [
{"role": "user", "content": "你好"}
],
"temperature": 0.7
}'
