# 安装依赖包
dnf install gcc make kernel-devel wget -y
# 升级系统
dnf update -y
# 重启服务器
reboot

# 禁用 Linux 驱动 nouveau，和英伟达官方驱动有冲突
grep "blacklist nouveau" /etc/modprobe.d/blacklist.conf || echo "blacklist nouveau" >> /etc/modprobe.d/blacklist.conf
# 内核中卸载该驱动
grep "options nouveau modeset=0" /etc/modprobe.d/blacklist.conf ||echo "options nouveau modeset=0" >> /etc/modprobe.d/blacklist.conf

dracut --force /boot/initramfs-$(uname -r).img
dracut --force --omit-drivers nouveau
reboot

# 检查 nouveau 是否关闭。没有结果说明已经关闭
lsmod | grep nouveau
# 安装驱动
chmod +x NVIDIA-Linux-*.run && ./NVIDIA-Linux-*.run -q -s --no-dkms --no-x-check 
# 安装完成后，重启服务器
reboot
nvidia-smi

# 关闭防火墙、selinux、dnsmasq、swap
systemctl disable --now firewalld 
systemctl disable --now dnsmasq
setenforce 0
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/sysconfig/selinux
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config

# 关闭 swap 分区：
swapoff -a && sysctl -w vm.swappiness=0
sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab

# 更改源及安装必备工具
sed -e 's|^mirrorlist=|#mirrorlist=|g' \
 -e 's|^#baseurl=http://dl.rockylinux.org/$contentdir|baseurl=https://mirrors.aliyun.com/rockylinux|g' \
 -i.bak \
 /etc/yum.repos.d/*.repo
 dnf makecache

yum install -y wget jq psmisc vim net-tools telnet yum-utils device-mapper-persistent-data lvm2 git

# docker安装
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

modprobe overlay
modprobe br_netfilter

cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF

sysctl --system
yum install docker-ce -y
systemctl enable --now docker

# 配置Nvidia Runtime，让docker识别GPU
curl -fsSL https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo | sudo tee /etc/yum.repos.d/nvidia-container-toolkit.repo

# 安装 Toolkit
dnf clean all
dnf makecache
dnf install -y nvidia-container-toolkit
# 生成 Docker 的 Nvidia 运行时
nvidia-ctk runtime configure --runtime=docker

# 查看生成的配置
cat /etc/docker/daemon.json
# 重启 Docker
systemctl restart docker

yum install -y python312
# 用python3.12创建虚拟环境
python3.12 -m venv modelscope
source modelscope/bin/activate

pip install modelscope -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple

modelscope download --model Qwen/Qwen3.5-4B --local_dir /data/models/Qwen3.5-4B

ls /data/models/Qwen3.5-4B/

mkdir /data/modelCompose/Qwen3.5-4B -p
cd /data/modelCompose/Qwen3.5-4B
vim docker-compose.yaml

version: '3.9'
services:
  qwen3.5-4b:
    shm_size: 16gb # 更改 docker 共享内存
    # m.daocloud.io/docker.io/vllm/vllm-openai:latest
    image: registry.cn-beijing.aliyuncs.com/dotbalo/vllm-openai:latest
    restart: always
    # served-model-name对外发布的名称，和模型名称保持一致
    command: --port 8080 --served-model-name Qwen3.5-4B --model /data/models/Qwen3.5-4B
    # 模型挂载进容器内的路径和外面保持一致即可
    volumes:
      - /data/models/Qwen3.5-4B:/data/models/Qwen3.5-4B
    healthcheck:
      test: ["CMD", "curl", "-s", "http://127.0.0.1:8080"]
      interval: 30s
      timeout: 2s
      retries: 2
      start_period: 60s
    ports:
      - 18080:8080
    environment:
      TZ: Asia/Shanghai
      LANG: C.UTF-8
      LC_ALL: C.UTF-8
      CUDA_VISIBLE_DEVICES: "0"
      NVIDIA_VISIBLE_DEVICES: "all"
    # 把宿主机的所有gpu卡挂载进来  
    deploy:
      resources:
        reservations:
          devices:
            - capabilities: [gpu]
              count: all

docker compose up -d

docker-compose ps

curl -X POST http://127.0.0.1:18080/v1/chat/completions -H "Content-Type: application/json" -d '{"model": "Qwen3.5-4B", "messages": [{"role": "user", "content": "介绍下你自己"}]}'

{"id":"chatcmpl-94d2a83f9d60c6cb","object":"chat.completion","created":1782302487,"model":"Qwen3.5-4B","choices":[{"index":0,"message":{"role":"assistant","content":"...你好！我是通义千问Qwen3.5，是阿里巴巴最新推出的通义实验室研发的**超大规模语言模型**，官方名\"通义千问\"，是阿里巴巴最新推出的通义千问语言模型，旨在为人类提供高质量、可信赖的 AI 服务。我支持**100+ 种语言**，具备**256K 原生长上下文**，支持**256K 上下文窗口**，可以精准处理超长文档或视频内容，还是**高效代码开发能力**（生成、调试、运行），都能灵活运用。..."},"finish_reason":"stop"}],"usage":{"prompt_tokens":13,"total_tokens":798,"completion_tokens":785}}

vim /data/modelCompose/Qwen3.5-4B/docker-compose.yaml

# --port 8080 --served-model-name Qwen3.5-4B --model /data/models/Qwen3.5-4B --gpu_memory_utilization 0.95

cd /data/modelCompose/Qwen3.5-4B/
docker compose down
docker compose up

version: '3.9'
services:
  qwen3.5-4b:
    shm_size: 16gb
    image: m.daocloud.io/docker.io/vllm/vllm-openai:latest
    restart: always
    command: --port 8080 --served-model-name Qwen3.5-4B --model /data/models/Qwen3.5-4B --gpu_memory_utilization 0.95 --tensor-parallel-size 2 # 设置张量并行为2，设置--tensor-parallel-size大于1时，vLLM会将模型的权重进行切分，并将这些分片均匀地分配到指定的多张 GPU 上。在推理时，这些 GPU 会协同计算，共同完成一次任务。
    volumes:
      - /data/models/Qwen3.5-4B:/data/models/Qwen3.5-4B
    healthcheck:
      test: ["CMD", "curl", "-s", "http://127.0.0.1:8080"]
      interval: 30s
      timeout: 2s
      retries: 2
      start_period: 60s
    ports:
      - 18080:8080
    environment:
      TZ: Asia/Shanghai
      LANG: C.UTF-8
      LC_ALL: C.UTF-8
      CUDA_VISIBLE_DEVICES: "0,1" # 使用 0 和 1 卡
      NVIDIA_VISIBLE_DEVICES: "all"
    deploy:
      resources:
        reservations:
          devices:
            - capabilities: [gpu]
              count: all

docker-compose down
docker-compose up -d

environment:
  NCCL_IB_DISABLE: "1"  # 禁用点对点通信，GPU之间的数据交换将转为通过系统内存和CPU中转
  NCCL_P2P_DISABLE: "1" # 禁用 IB 网络，让 NCCL 使用最通用的 PCIe 进行通信

--port 8080 --served-model-name Qwen3.5-4B --model /data/models/Qwen3.5-4B --gpu_memory_utilization 0.95 --max-model-len 65536 --max-num-batched-tokens 65536

[model.py:1554] Using max model len 65536

--port 8080 --served-model-name Qwen3.5-4B --model /data/models/Qwen3.5-4B --gpu_memory_utilization 0.6 --tensor-parallel-size 2 --max-model-len 65536 --max-num-batched-tokens 65536 --api-key "xxxx"

curl -H "Authorization: Bearer xxxx" -X POST http://127.0.0.1:18080/v1/chat/completions -H "Content-Type: application/json" -d '{"model": "Qwen3.5-4B","stream": true, "messages": [{"role": "user", "content": "介绍下你自己"}]}'

source modelscope/bin/activate
modelscope download --model Qwen/Qwen3-Reranker-4B --local_dir /data/models/Qwen3-Reranker-4B
modelscope download --model Qwen/Qwen3-Embedding-4B --local_dir /data/models/Qwen3-Embedding-4B

mkdir /data/modelCompose/Qwen3-Embedding-4B -p
cd /data/modelCompose/Qwen3-Embedding-4B
vim docker-compose.yaml

version: '3.9'
services:
  qwen3-embedding-4b:
    shm_size: 16gb
    image: registry.cn-beijing.aliyuncs.com/dotbalo/vllm-openai:latest
    restart: always
    command: --port 8080 --served-model-name Qwen3-Embedding-4B --model /data/models/Qwen3-Embedding-4B --gpu_memory_utilization 0.5 --tensor-parallel-size 1 --max-model-len 8192 --max-num-batched-tokens 8192 --api-key xxxx
    volumes:
      - /data/models/Qwen3-Embedding-4B:/data/models/Qwen3-Embedding-4B
    healthcheck:
      test: ["CMD", "curl", "-s", "http://127.0.0.1:8080"]
      interval: 30s
      timeout: 2s
      retries: 2
      start_period: 60s
    ports:
      - 18080:8080
    environment:
      TZ: Asia/Shanghai
      LANG: C.UTF-8
      LC_ALL: C.UTF-8
      CUDA_VISIBLE_DEVICES: "0"
      NVIDIA_VISIBLE_DEVICES: "all"
    deploy:
      resources:
        reservations:
          devices:
            - capabilities: [gpu]
              count: all

curl http://localhost:18080/v1/embeddings \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer xxxx" \
  -d '{
    "model": "Qwen3-Embedding-4B",
    "input": ["人工智能正在改变世界", "vLLM 是一个高性能推理引擎"]
  }'

{"id":"embd-89098f2ed7063712","object":"list","created":1782530142,"model":"Qwen3-Embedding-4B","data":[{"index":0,"object":"embedding","embedding":[-0.0005068916361778975...

mkdir /data/modelCompose/Qwen3-Reranker-4B -p
cd /data/modelCompose/Qwen3-Reranker-4B
vim docker-compose.yaml

version: '3.9'
services:
  qwen3-reranker-4b:
    shm_size: 16gb
    image: registry.cn-beijing.aliyuncs.com/dotbalo/vllm-openai:latest
    restart: always
    command: --port 8080 --served-model-name Qwen3-Reranker-4B --model /data/models/Qwen3-Reranker-4B --gpu_memory_utilization 0.5 --tensor-parallel-size 1 --max-model-len 32768 --max-num-batched-tokens 32768 --api-key xxxx --hf-overrides '{"architectures":["Qwen3ForSequenceClassification"],"classifier_from_token":["yes","no"],"is_original_qwen3_reranker":true}'
    volumes:
      - /data/models/Qwen3-Reranker-4B:/data/models/Qwen3-Reranker-4B
    healthcheck:
      test: ["CMD", "curl", "-s", "http://127.0.0.1:8080"]
      interval: 30s
      timeout: 2s
      retries: 2
      start_period: 60s
    ports:
      - 18080:8080
    environment:
      TZ: Asia/Shanghai
      LANG: C.UTF-8
      LC_ALL: C.UTF-8
      CUDA_VISIBLE_DEVICES: "0"
      NVIDIA_VISIBLE_DEVICES: "all"
    deploy:
      resources:
        reservations:
          devices:
            - capabilities: [gpu]
              count: all

curl http://localhost:18080/v1/rerank \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer xxxx" \
  -d '{
    "model": "Qwen3-Reranker-4B",
    "query": "如何修改密码？",
    "documents": [
      "点击设置 > 安全 > 修改密码即可更新凭据",
      "点击忘记密码，然后通过邮箱修改密码？",
      "今天天气不错"
    ]
  }'

{"id":"score-8e193251b12aa95b","model":"Qwen3-Reranker-4B","usage":{"prompt_tokens":38,"total_tokens":38},"results":[{"index":0,"document":{"text":"点击设置 > 安全 > 修改密码即可更新凭据","multi_modal":null},"relevance_score":0.5528110265731812},{"index":1,"document":{"text":"点击忘记密码，然后通过邮箱修改密码？","multi_modal":null},"relevance_score":0.2518998980522156},{"index":2,"document":{"text":"今天天气不错","multi_modal":null},"relevance_score":0.05458692088723183}]}

curl http://localhost:18080/v1/rerank \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer xxxx" \
  -d '{
    "model": "Qwen3-Reranker-4B",
    "query": "如何修改密码？",
    "documents": [
      "点击设置 > 安全 > 修改密码即可更新凭据",
      "点击忘记密码，然后通过邮箱修改密码？",
      "今天天气不错"
    ],
    "top_n": 2
  }'

mkdir -p litellm
vim docker-compose.yaml

services:
  litellm:
    image: docker.litellm.ai/berriai/litellm:main-stable
    ports:
      - "4000:4000" # Map the container port to the host, change the host port if necessary
    environment:
      DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
      STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI
      LITELLM_MASTER_KEY: litellm_password
    depends_on:
      - db
    healthcheck:
      test:
        - CMD-SHELL
        - python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')" # Command to execute for health check
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s

  db:
    image: postgres:17.2
    restart: always
    container_name: litellm_db
    environment:
      POSTGRES_DB: litellm
      POSTGRES_USER: llmproxy
      POSTGRES_PASSWORD: dbpassword9090
    #ports:
    #  - "5432:5432"
    volumes:
      - ./data/postgres_data:/var/lib/postgresql/data # Persists Postgres data across container restarts
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -d litellm -U llmproxy"]
      interval: 1s
      timeout: 5s
      retries: 10

docker compose pull

docker compose up -d

docker logs -f litellm-litellm-1

Virtual Key: sk-cRyZdFtJ8WSDZxpjEAUxTw

# curl -H "Authorization: Bearer sk-cRyZdFtJ8WSDZxpjEAUxTw" -X POST http://127.0.0.1:4000/v1/chat/completions -H "Content-Type: application/json" -d '{"model": "Qwen3.5-4B","stream": true, "messages": [{"role": "user", "content": "介绍下你自己"}]}'

curl -H "Authorization: Bearer sk-gPy_Dc2avf6sH8wWwu8VRw" -X POST http://127.0.0.1:4000/v1/chat/completions -H "Content-Type: application/json" -d '{"model": "Qwen3.5-4B","stream": true, "messages": [{"role": "user", "content": "介绍下你自己"}]}'

{"error":{"message":"team not allowed to access model. This team can only access models=['deepseek-v4-pro']. Tried to access Qwen3.5-4B","type":"team_model_access_denied","param":"model","code":"403"}}

curl -H "Authorization: Bearer sk-1k_6cjksNc0vo13LqZa40g" -X POST http://127.0.0.1:4000/v1/chat/completions -H "Content-Type: application/json" -d '{"model": "Qwen3.5-4B","stream": true, "messages": [{"role": "user", "content": "介绍下你自己"}]}'

{"error":{"message":"team not allowed to access model. This team can only access models=['deepseek-v4-pro']. Tried to access Qwen3.5-4B","type":"team_model_access_denied","param":"model","code":"403"}}

curl -H "Authorization: Bearer sk-An1rHO3RWiA7LDh4jA8rtQ" -X POST http://127.0.0.1:4000/v1/chat/completions -H "Content-Type: application/json" -d '{"model": "Qwen3.5-4B","stream": true, "messages": [{"role": "user", "content": "列举一下北京比较著名的景点有哪些？"}]}'

curl -H "Authorization: Bearer xxx" -X POST http://127.0.0.1:4000/v1/chat/completions -H "Content-Type: application/json" -d '{"model": "Qwen3.5-4B","stream": true, "messages": [{"role": "user", "content": "列举一下北京比较著名的景点有哪些？"}]}'

{"error":{"message":"Budget has been exceeded! Team=9cbca9c9-53f1-4387-bbb4-a0c8a7b21530 Current cost: 0.03890300000000001, Max budget: 0.02","type":"budget_exceeded","param":null,"code":"429"}}