LLM 生产服务化：vLLM 部署与 GPU 推理优化实战

pip install vllm

# 部署 Qwen3-72B，4 卡张量并行
vllm serve Qwen/Qwen3-72B-Instruct \
  --tensor-parallel-size 4 \
  --gpu-memory-utilization 0.9 \
  --max-model-len 32768 \
  --served-model-name qwen3-72b \
  --host 0.0.0.0 \
  --port 8000

from openai import OpenAI

client = OpenAI(
    base_url="http://your-vllm-host:8000/v1",
    api_key="not-needed",  # vLLM 默认不校验 key
)

response = client.chat.completions.create(
    model="qwen3-72b",
    messages=[
        {"role": "system", "content": "你是一个专业的代码审查助手"},
        {"role": "user", "content": "请帮我 review 这段 Python 代码：\n```python\ndef add(a, b):\n    return a + b\n```"}
    ],
    temperature=0.3,
    max_tokens=1024,
)

print(response.choices[0].message.content)

stream = client.chat.completions.create(
    model="qwen3-72b",
    messages=[{"role": "user", "content": "写一首关于工程师的诗"}],
    stream=True,
)

for chunk in stream:
    if chunk.choices[0].delta.content:
        print(chunk.choices[0].delta.content, end="", flush=True)

kubectl apply -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v0.16.0/deployments/static/nvidia-device-plugin.yml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm-qwen3-72b
  namespace: ai-inference
spec:
  replicas: 1
  selector:
    matchLabels:
      app: vllm-qwen3-72b
  template:
    metadata:
      labels:
        app: vllm-qwen3-72b
    spec:
      # 节点亲和性：只调度到有 GPU 的节点
      nodeSelector:
        nvidia.com/gpu.present: "true"
        node.kubernetes.io/instance-type: "p4d.24xlarge"  # 8x A100-40G
      tolerations:
        - key: "nvidia.com/gpu"
          operator: "Exists"
          effect: "NoSchedule"
      containers:
        - name: vllm
          image: vllm/vllm-openai:v0.7.3
          command:
            - python
            - -m
            - vllm.entrypoints.openai.api_server
          args:
            - --model
            - /models/Qwen3-72B-Instruct
            - --tensor-parallel-size
            - "4"
            - --gpu-memory-utilization
            - "0.9"
            - --max-model-len
            - "32768"
            - --served-model-name
            - qwen3-72b
            - --host
            - "0.0.0.0"
            - --port
            - "8000"
          ports:
            - containerPort: 8000
              name: http
          resources:
            requests:
              cpu: "8"
              memory: "64Gi"
              nvidia.com/gpu: "4"   # 申请 4 张 GPU
            limits:
              cpu: "16"
              memory: "128Gi"
              nvidia.com/gpu: "4"
          volumeMounts:
            - name: model-storage
              mountPath: /models
          env:
            - name: HUGGING_FACE_HUB_TOKEN
              valueFrom:
                secretKeyRef:
                  name: hf-token
                  key: token
          # 启动探针：vLLM 加载 70B 模型需要 5-10 分钟
          startupProbe:
            httpGet:
              path: /health
              port: 8000
            failureThreshold: 60
            periodSeconds: 15
          livenessProbe:
            httpGet:
              path: /health
              port: 8000
            initialDelaySeconds: 30
            periodSeconds: 30
          readinessProbe:
            httpGet:
              path: /health
              port: 8000
            periodSeconds: 10
      volumes:
        - name: model-storage
          persistentVolumeClaim:
            claimName: model-storage-pvc
---
apiVersion: v1
kind: Service
metadata:
  name: vllm-qwen3-72b
  namespace: ai-inference
spec:
  selector:
    app: vllm-qwen3-72b
  ports:
    - port: 80
      targetPort: 8000
  type: ClusterIP

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: model-storage-pvc
  namespace: ai-inference
spec:
  accessModes:
    - ReadWriteMany   # EFS 支持多节点挂载
  storageClassName: efs-sc
  resources:
    requests:
      storage: 200Gi  # Qwen3-72B FP16 约 144GB

vllm serve Qwen/Qwen3-72B-Instruct \
  --speculative-model Qwen/Qwen3-7B-Instruct \
  --num-speculative-tokens 5 \
  --tensor-parallel-size 4

# AWQ 量化，模型大小减半，精度损失很小
vllm serve Qwen/Qwen3-72B-Instruct-AWQ \
  --quantization awq \
  --tensor-parallel-size 2  # 量化后只需 2 卡

# 关键指标
vllm:num_requests_running         # 当前正在处理的请求数
vllm:num_requests_waiting         # 等待队列中的请求数
vllm:gpu_cache_usage_perc         # KV Cache 使用率
vllm:time_to_first_token_seconds  # TTFT 分布
vllm:time_per_output_token_seconds # 每个 output token 的时间（=1/TPS）
vllm:e2e_request_latency_seconds  # 端到端延迟

- job_name: 'vllm'
  static_configs:
    - targets: ['vllm-service:80']
  metrics_path: '/metrics'

- alert: VLLMHighQueueDepth
  expr: vllm:num_requests_waiting > 20
  for: 1m
  annotations:
    summary: "vLLM 请求队列积压，考虑扩容"

- alert: VLLMHighTTFT
  expr: histogram_quantile(0.99, vllm:time_to_first_token_seconds_bucket) > 10
  for: 2m
  annotations:
    summary: "P99 TTFT 超过 10 秒，服务质量下降"